def translate_criteria(self, model, crit):
"""
To preserve backwards compatability of our search API, we must translate the fields from
the external representation to the internal representation. This is done most often with
'id' since this is not an allowable key in the database in Mongoengine.
This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
be a dictionary containing the fields that have been renamed in the format:
{'internal repr': 'external repr'}
:param model: the class that defines this document's fields
:type model: sublcass of mongoengine.Document
:param crit: criteria object to be translated from external to internal representation
:type crit: pulp.server.db.model.criteria.Criteria
:return: translated Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
# Circular import avoidance, since criteria imports models which imports serializers
from pulp.server.db.model.criteria import Criteria
crit_dict = crit.as_dict()
if crit.filters:
crit_dict['filters'] = self.translate_filters(model, crit.filters)
if crit.sort:
sort = [(self.translate_field(model, field), direc)
for field, direc in crit.sort]
crit_dict['sort'] = sort
if crit.fields:
crit_dict['fields'] = [
self.translate_field(model, field) for field in crit.fields
]
return Criteria.from_dict(crit_dict)
def translate_criteria(self, model, crit):
"""
To preserve backwards compatability of our search API, we must translate the fields from
the external representation to the internal representation. This is done most often with
'id' since this is not an allowable key in the database in Mongoengine.
This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
be a dictionary containing the fields that have been renamed in the format:
{'internal repr': 'external repr'}
:param model: the class that defines this document's fields
:type model: sublcass of mongoengine.Document
:param crit: criteria object to be translated from external to internal representation
:type crit: pulp.server.db.model.criteria.Criteria
:return: translated Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
# Circular import avoidance, since criteria imports models which imports serializers
from pulp.server.db.model.criteria import Criteria
crit_dict = crit.as_dict()
if crit.filters:
crit_dict['filters'] = self._translate_filters(model, crit.filters)
if crit.sort:
sort = [(self._translate(model, field), direc) for field, direc in crit.sort]
crit_dict['sort'] = sort
if crit.fields:
crit_dict['fields'] = [self._translate(model, field) for field in crit.fields]
return Criteria.from_dict(crit_dict)
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async(
(repo_id, batch), **{'group_id': task_group_id})
return task_group_id
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ["id"]
repo_ids = [r["id"] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find({"repo_id": repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability["profile_hash"]
unit_profile = UnitProfile.get_collection().find_one(
{"profile_hash": profile_hash}, fields=["id", "content_type"]
)
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile["content_type"], unit_profile["id"], repo_id, existing_applicability
)
def regenerate_applicability_for_consumers(consumer_criteria):
"""
Regenerate and save applicability data for given updated consumers.
:param consumer_criteria: The consumer selection criteria
:type consumer_criteria: dict
"""
consumer_criteria = Criteria.from_dict(consumer_criteria)
consumer_query_manager = managers.consumer_query_manager()
# Process consumer_criteria and get all the consumer ids satisfied by the criteria
consumer_criteria.fields = ['id']
consumer_ids = [
c['id']
for c in consumer_query_manager.find_by_criteria(consumer_criteria)
]
# Following logic of checking existing applicability and getting required data
# to generate applicability is a bit more complicated than what it could be 'by design'.
# It is to optimize the number of db queries and improving applicability generation
# performance. Please consider the implications for applicability generation time
# when making any modifications to this code.
consumer_profile_map = ApplicabilityRegenerationManager._get_consumer_profile_map(
consumer_ids)
repo_consumer_map = ApplicabilityRegenerationManager._get_repo_consumer_map(
consumer_ids=consumer_ids)
# Since there could be different types of profiles and they are related (at the moment
# these are RPMs and Modulemds in the RPM plugin), it's important to calculate applicability
# not per profile but for a combination of all profiles of one consumer,
# all_profiles_hash identifies that set of profiles.
# Iterate through each unique all_profiles_hash and regenerate applicability,
# if it doesn't exist.
for repo_id in repo_consumer_map:
seen_hashes = set()
for consumer_id in repo_consumer_map[repo_id]:
if consumer_id in consumer_profile_map:
all_profiles_hash = consumer_profile_map[consumer_id][
'all_profiles_hash']
if all_profiles_hash in seen_hashes:
continue
profiles = consumer_profile_map[consumer_id]['profiles']
seen_hashes.add(all_profiles_hash)
if ApplicabilityRegenerationManager._is_existing_applicability(
repo_id, all_profiles_hash):
continue
# If applicability does not exist, generate applicability data for given
# profiles and repo id.
ApplicabilityRegenerationManager.regenerate_applicability(
all_profiles_hash, profiles, repo_id)
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
repo_consumer_map = ApplicabilityRegenerationManager._get_repo_consumer_map(
repo_ids=repo_ids)
consumer_ids = itertools.chain(*repo_consumer_map.values())
consumer_profile_map = ApplicabilityRegenerationManager._get_consumer_profile_map(
consumer_ids)
task_group_id = uuid4()
batch_size = 10
# list of tuples (repo_id, all_profiles_hash, profiles)
profiles_to_process = []
for repo_id in repo_consumer_map:
seen_hashes = set()
for consumer_id in repo_consumer_map[repo_id]:
if consumer_id in consumer_profile_map:
all_profiles_hash = consumer_profile_map[consumer_id][
'all_profiles_hash']
if all_profiles_hash in seen_hashes:
continue
seen_hashes.add(all_profiles_hash)
profiles = consumer_profile_map[consumer_id]['profiles']
if len(profiles_to_process) < batch_size:
profiles_to_process.append(
(repo_id, all_profiles_hash, profiles))
else:
batch_regenerate_applicability_task.apply_async(
(profiles_to_process, ),
**{'group_id': task_group_id})
profiles_to_process = []
# last few non-processed profiles which didn't make a whole batch
if profiles_to_process:
batch_regenerate_applicability_task.apply_async(
(profiles_to_process, ), **{'group_id': task_group_id})
return task_group_id
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
# the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
# more details.
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({
'repo_id': repo_id
}).batch_size(5)
for existing_applicability in existing_applicabilities:
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async((repo_id, batch),
**{'group_id': task_group_id})
return task_group_id
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
# the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
# more details.
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}).batch_size(5)
for existing_applicability in existing_applicabilities:
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
projection=['id',
'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
repo_consumer_map = ApplicabilityRegenerationManager._get_repo_consumer_map(
repo_ids=repo_ids)
consumer_ids = itertools.chain(*repo_consumer_map.values())
consumer_profile_map = ApplicabilityRegenerationManager._get_consumer_profile_map(
consumer_ids)
for repo_id in repo_consumer_map:
seen_hashes = set()
for consumer_id in repo_consumer_map[repo_id]:
if consumer_id in consumer_profile_map:
all_profiles_hash = consumer_profile_map[consumer_id][
'all_profiles_hash']
if all_profiles_hash in seen_hashes:
continue
seen_hashes.add(all_profiles_hash)
profiles = consumer_profile_map[consumer_id]['profiles']
# Regenerate applicability data for a given all_profiles_hash and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
all_profiles_hash, profiles, repo_id)
def regenerate_applicability_for_consumers(consumer_criteria):
"""
Regenerate and save applicability data for given updated consumers.
:param consumer_criteria: The consumer selection criteria
:type consumer_criteria: dict
"""
consumer_criteria = Criteria.from_dict(consumer_criteria)
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
consumer_profile_manager = managers.consumer_profile_manager()
# Process consumer_criteria and get all the consumer ids satisfied by the criteria
consumer_criteria.fields = ['id']
consumer_ids = [c['id'] for c in consumer_query_manager.find_by_criteria(consumer_criteria)]
# Following logic of checking existing applicability and getting required data
# to generate applicability is a bit more complicated than what it could be 'by design'.
# It is to optimize the number of db queries and improving applicability generation
# performance. Please consider the implications for applicability generation time
# when making any modifications to this code.
# Get all unit profiles associated with given consumers
unit_profile_criteria = Criteria(
filters={'consumer_id': {'$in': consumer_ids}},
fields=['consumer_id', 'profile_hash', 'content_type', 'id'])
all_unit_profiles = consumer_profile_manager.find_by_criteria(unit_profile_criteria)
# Create a consumer-profile map with consumer id as the key and list of tuples
# with profile details as the value
consumer_unit_profiles_map = {}
# Also create a map of profile_id keyed by profile_hash for profile lookup.
profile_hash_profile_id_map = {}
for unit_profile in all_unit_profiles:
profile_hash = unit_profile['profile_hash']
content_type = unit_profile['content_type']
consumer_id = unit_profile['consumer_id']
profile_id = unit_profile['id']
profile_tuple = (profile_hash, content_type)
# Add this tuple to the list of profile tuples for a consumer
consumer_unit_profiles_map.setdefault(consumer_id, []).append(profile_tuple)
# We need just one profile_id per profile_hash to be used in regenerate_applicability
# method to get the actual profile corresponding to given profile_hash.
if profile_hash not in profile_hash_profile_id_map:
profile_hash_profile_id_map[profile_hash] = profile_id
# Get all repos bound to given consumers
bind_criteria = Criteria(filters={'consumer_id': {'$in': consumer_ids}},
fields=['repo_id', 'consumer_id'])
all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)
# Create a repo-consumer map with repo_id as the key and consumer_id list as the value
repo_consumers_map = {}
for binding in all_repo_bindings:
repo_consumers_map.setdefault(binding['repo_id'], []).append(binding['consumer_id'])
# Create a set of (repo_id, (profile_hash, content_type))
repo_profile_hashes = set()
for repo_id, consumer_id_list in repo_consumers_map.items():
for consumer_id in consumer_id_list:
if consumer_id in consumer_unit_profiles_map:
for unit_profile_tuple in consumer_unit_profiles_map[consumer_id]:
repo_profile_hashes.add((repo_id, unit_profile_tuple))
# Iterate through each tuple in repo_profile_hashes set and regenerate applicability,
# if it doesn't exist. These are all guaranteed to be unique tuples because of the logic
# used to create maps and sets above, eliminating multiple unnecessary queries
# to check for existing applicability for same profiles.
manager = managers.applicability_regeneration_manager()
for repo_id, (profile_hash, content_type) in repo_profile_hashes:
# Check if applicability for given profile_hash and repo_id already exists
if ApplicabilityRegenerationManager._is_existing_applicability(repo_id, profile_hash):
continue
# If applicability does not exist, generate applicability data for given profile
# and repo id.
profile_id = profile_hash_profile_id_map[profile_hash]
manager.regenerate_applicability(profile_hash, content_type, profile_id, repo_id)
def regenerate_applicability_for_consumers(consumer_criteria):
"""
Regenerate and save applicability data for given updated consumers.
:param consumer_criteria: The consumer selection criteria
:type consumer_criteria: dict
"""
consumer_criteria = Criteria.from_dict(consumer_criteria)
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
consumer_profile_manager = managers.consumer_profile_manager()
# Process consumer_criteria and get all the consumer ids satisfied by the criteria
consumer_criteria.fields = ['id']
consumer_ids = [
c['id']
for c in consumer_query_manager.find_by_criteria(consumer_criteria)
]
# Following logic of checking existing applicability and getting required data
# to generate applicability is a bit more complicated than what it could be 'by design'.
# It is to optimize the number of db queries and improving applicability generation
# performance. Please consider the implications for applicability generation time
# when making any modifications to this code.
# Get all unit profiles associated with given consumers
unit_profile_criteria = Criteria(
filters={'consumer_id': {
'$in': consumer_ids
}},
fields=['consumer_id', 'profile_hash', 'content_type', 'id'])
all_unit_profiles = consumer_profile_manager.find_by_criteria(
unit_profile_criteria)
# Create a consumer-profile map with consumer id as the key and list of tuples
# with profile details as the value
consumer_unit_profiles_map = {}
# Also create a map of profile_id keyed by profile_hash for profile lookup.
profile_hash_profile_id_map = {}
for unit_profile in all_unit_profiles:
profile_hash = unit_profile['profile_hash']
content_type = unit_profile['content_type']
consumer_id = unit_profile['consumer_id']
profile_id = unit_profile['id']
profile_tuple = (profile_hash, content_type)
# Add this tuple to the list of profile tuples for a consumer
consumer_unit_profiles_map.setdefault(consumer_id,
[]).append(profile_tuple)
# We need just one profile_id per profile_hash to be used in regenerate_applicability
# method to get the actual profile corresponding to given profile_hash.
if profile_hash not in profile_hash_profile_id_map:
profile_hash_profile_id_map[profile_hash] = profile_id
# Get all repos bound to given consumers
bind_criteria = Criteria(
filters={'consumer_id': {
'$in': consumer_ids
}},
fields=['repo_id', 'consumer_id'])
all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)
# Create a repo-consumer map with repo_id as the key and consumer_id list as the value
repo_consumers_map = {}
for binding in all_repo_bindings:
repo_consumers_map.setdefault(binding['repo_id'],
[]).append(binding['consumer_id'])
# Create a set of (repo_id, (profile_hash, content_type))
repo_profile_hashes = set()
for repo_id, consumer_id_list in repo_consumers_map.items():
for consumer_id in consumer_id_list:
if consumer_id in consumer_unit_profiles_map:
for unit_profile_tuple in consumer_unit_profiles_map[
consumer_id]:
repo_profile_hashes.add((repo_id, unit_profile_tuple))
# Iterate through each tuple in repo_profile_hashes set and regenerate applicability,
# if it doesn't exist. These are all guaranteed to be unique tuples because of the logic
# used to create maps and sets above, eliminating multiple unnecessary queries
# to check for existing applicability for same profiles.
manager = managers.applicability_regeneration_manager()
for repo_id, (profile_hash, content_type) in repo_profile_hashes:
# Check if applicability for given profile_hash and repo_id already exists
if ApplicabilityRegenerationManager._is_existing_applicability(
repo_id, profile_hash):
continue
# If applicability does not exist, generate applicability data for given profile
# and repo id.
profile_id = profile_hash_profile_id_map[profile_hash]
manager.regenerate_applicability(profile_hash, content_type,
profile_id, repo_id)
