def setUp(self):
base.PulpServerTests.setUp(self)
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
plugins._create_manager()
mock_plugins.install()
profiler, cfg = plugins.get_profiler_by_type('rpm')
profiler.find_applicable_units = \
Mock(side_effect=lambda i,r,t,u,c,x:
[ApplicabilityReport('mysummary', 'mydetails')])
def tearDown(self):
super(BaseProfilerConduitTests, self).tearDown()
Consumer.get_collection().remove()
Repo.get_collection().remove()
RepoDistributor.get_collection().remove()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
typedb.clean()
factory.reset()
def tearDown(self):
super(BaseProfilerConduitTests, self).tearDown()
Consumer.get_collection().remove()
Repo.get_collection().remove()
RepoDistributor.get_collection().remove()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
typedb.clean()
factory.reset()
def setUp(self):
super(BaseProfilerConduitTests, self).setUp()
Consumer.get_collection().remove()
RepoDistributor.get_collection().remove()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
plugin_api._create_manager()
typedb.update_database([self.TYPE_1_DEF, self.TYPE_2_DEF])
mock_plugins.install()
def setUp(self):
base.PulpServerTests.setUp(self)
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
plugins._create_manager()
mock_plugins.install()
profiler, cfg = plugins.get_profiler_by_type('rpm')
profiler.units_applicable = \
Mock(side_effect=lambda i,r,t,u,c,x:
[ApplicabilityReport('mysummary', 'mydetails')])
def setUp(self):
base.PulpWebserviceTests.setUp(self)
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
plugin_api._create_manager()
mock_plugins.install()
profiler = plugin_api.get_profiler_by_type('errata')[0]
profiler.unit_applicable = \
mock.Mock(side_effect=lambda i,u,c,x:
ApplicabilityReport(u, True, self.SUMMARY, self.DETAILS))
def setUp(self):
super(BaseProfilerConduitTests, self).setUp()
Consumer.get_collection().remove()
RepoDistributor.get_collection().remove()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
plugin_api._create_manager()
typedb.update_database([self.TYPE_1_DEF, self.TYPE_2_DEF])
mock_plugins.install()
def tearDown(self):
super(BaseProfilerConduitTests, self).tearDown()
Consumer.get_collection().remove()
model.Repository.objects.delete()
model.Distributor.objects.delete()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
typedb.clean()
factory.reset()
mock_plugins.reset()
def tearDown(self):
super(BaseProfilerConduitTests, self).tearDown()
Consumer.get_collection().remove()
model.Repository.objects.delete()
model.Distributor.objects.delete()
Bind.get_collection().remove()
RepoContentUnit.get_collection().remove()
UnitProfile.get_collection().remove()
typedb.clean()
factory.reset()
mock_plugins.reset()
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ["id"]
repo_ids = [r["id"] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find({"repo_id": repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability["profile_hash"]
unit_profile = UnitProfile.get_collection().find_one(
{"profile_hash": profile_hash}, fields=["id", "content_type"]
)
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile["content_type"], unit_profile["id"], repo_id, existing_applicability
)
def _add_profiles_to_consumer_map_and_get_hashes(consumer_ids, consumer_map):
"""
Query for all the profiles associated with the given list of consumer_ids, add those
profiles to the consumer_map, and then return a list of all profile_hashes.
:param consumer_ids: A list of consumer_ids that we want to map the profiles to
:type consumer_ids: list
:param consumer_map: A dictionary mapping consumer_ids to a dictionary with key 'profiles',
which indexes a list that this method will append the found profiles
to.
:type consumer_map: dict
:return: A list of the profile_hashes that were associated with the given
consumers
:rtype: list
"""
profiles = UnitProfile.get_collection().find(
{'consumer_id': {
'$in': consumer_ids
}},
projection=['consumer_id', 'profile_hash'])
profile_hashes = set()
for p in profiles:
consumer_map[p['consumer_id']]['profiles'].append(p)
profile_hashes.add(p['profile_hash'])
# Let's return a list of all the unique profile_hashes for the query we will do a
# bit later for applicability data
return list(profile_hashes)
def update(self, consumer_id, content_type, profile):
"""
Update a unit profile.
Created if not already exists.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
@param profile: The unit profile
@type profile: object
"""
try:
profiler, config = plugin_api.get_profiler_by_type(content_type)
except plugin_exceptions.PluginNotFound:
# Not all profile types have a type specific profiler, so let's use the baseclass
# Profiler
profiler, config = (Profiler(), {})
consumer = factory.consumer_manager().get_consumer(consumer_id)
# Allow the profiler a chance to update the profile before we save it
profile = profiler.update_profile(consumer, content_type, profile, config)
try:
p = self.get_profile(consumer_id, content_type)
p['profile'] = profile
# We store the profile's hash anytime the profile gets altered
p['profile_hash'] = UnitProfile.calculate_hash(profile)
except MissingResource:
p = UnitProfile(consumer_id, content_type, profile)
collection = UnitProfile.get_collection()
collection.save(p, safe=True)
return p
def batch_regenerate_applicability(repo_id, profile_hashes):
"""
Regenerate and save applicability data for a batch of existing applicabilities
:param repo_id: Repository id for which applicability is being calculated
:type repo_id: str
:param profile_hashes: Tuple of consumer profile hashes for applicability profiles.
Don't pass too much of these, all the profile data
associated with these hashes is loaded into the memory.
:type profile_hashes: tuple of dicts in form of {'profile_hash': str}
"""
profile_hash_list = [phash['profile_hash'] for phash in profile_hashes]
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{"repo_id": repo_id, "profile_hash": {"$in": profile_hash_list}})
for existing_applicability in list(existing_applicabilities):
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def test_consumer_deleted(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 2)
# Test
manager.consumer_deleted(self.CONSUMER_ID)
cursor = collection.find()
profiles = list(cursor)
self.assertEquals(len(profiles), 0)
def _add_profiles_to_consumer_map_and_get_hashes(consumer_ids, consumer_map):
"""
Query for all the profiles associated with the given list of consumer_ids, add those
profiles to the consumer_map, and then return a list of all profile_hashes.
:param consumer_ids: A list of consumer_ids that we want to map the profiles to
:type consumer_ids: list
:param consumer_map: A dictionary mapping consumer_ids to a dictionary with key 'profiles',
which indexes a list that this method will append the found profiles
to.
:type consumer_map: dict
:return: A list of the profile_hashes that were associated with the given
consumers
:rtype: list
"""
profiles = UnitProfile.get_collection().find(
{'consumer_id': {'$in': consumer_ids}},
projection=['consumer_id', 'profile_hash'])
profile_hashes = set()
for p in profiles:
consumer_map[p['consumer_id']]['profiles'].append(p)
profile_hashes.add(p['profile_hash'])
# Let's return a list of all the unique profile_hashes for the query we will do a
# bit later for applicability data
return list(profile_hashes)
def remove_orphans():
"""
The RepoProfileApplicability objects can become orphaned over time, as repositories are
deleted, or as consumer profiles change. This method searches for RepoProfileApplicability
objects that reference either repositories or profile hashes that no longer exist in Pulp.
"""
# Find all of the repo_ids that are referenced by RepoProfileApplicability objects
rpa_collection = RepoProfileApplicability.get_collection()
rpa_repo_ids = rpa_collection.distinct('repo_id')
# Find all of the repo_ids that exist in Pulp
repo_ids = model.Repository.objects.distinct('repo_id')
# Find rpa_repo_ids that aren't part of repo_ids
missing_repo_ids = list(set(rpa_repo_ids) - set(repo_ids))
# Remove all RepoProfileApplicability objects that reference these repo_ids
if missing_repo_ids:
rpa_collection.remove({'repo_id': {'$in': missing_repo_ids}})
# Next, we need to find profile_hashes that don't exist in the UnitProfile collection
rpa_profile_hashes = rpa_collection.distinct('profile_hash')
# Find the profile hashes that exist in current UnitProfiles
profile_hashes = UnitProfile.get_collection().distinct('profile_hash')
# Find profile hashes that we have RepoProfileApplicability objects for, but no real
# UnitProfiles
missing_profile_hashes = list(set(rpa_profile_hashes) - set(profile_hashes))
# Remove all RepoProfileApplicability objects that reference these profile hashes
if missing_profile_hashes:
rpa_collection.remove({'profile_hash': {'$in': missing_profile_hashes}})
def update(consumer_id, content_type, profile):
"""
Update a unit profile.
Created if not already exists.
:param consumer_id: uniquely identifies the consumer.
:type consumer_id: str
:param content_type: The profile (content) type ID.
:type content_type: str
:param profile: The unit profile
:type profile: object
"""
try:
profiler, config = plugin_api.get_profiler_by_type(content_type)
except plugin_exceptions.PluginNotFound:
# Not all profile types have a type specific profiler, so let's use the baseclass
# Profiler
profiler, config = (Profiler(), {})
consumer = factory.consumer_manager().get_consumer(consumer_id)
# Allow the profiler a chance to update the profile before we save it
profile = profiler.update_profile(consumer, content_type, profile,
config)
try:
p = ProfileManager.get_profile(consumer_id, content_type)
p['profile'] = profile
# We store the profile's hash anytime the profile gets altered
p['profile_hash'] = UnitProfile.calculate_hash(profile)
except MissingResource:
p = UnitProfile(consumer_id, content_type, profile)
collection = UnitProfile.get_collection()
collection.save(p, safe=True)
return p
def test_consumer_deleted(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
collection = UnitProfile.get_collection()
cursor = collection.find({"consumer_id": self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 2)
# Test
manager.consumer_deleted(self.CONSUMER_ID)
cursor = collection.find()
profiles = list(cursor)
self.assertEquals(len(profiles), 0)
def regenerate_applicability(profile_hash, content_type, profile_id, bound_repo_id, existing_applicability=None):
"""
Regenerate and save applicability data for given profile and bound repo id.
If existing_applicability is not None, replace it with the new applicability data.
:param profile_hash: hash of the unit profile
:type profile_hash: basestring
:param content_type: profile (unit) type ID
:type content_type: str
:param profile_id: unique id of the unit profile
:type profile_id: str
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
:param existing_applicability: existing RepoProfileApplicability object to be replaced
:type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given unit_profile
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if set(repo_content_types) & set(profiler.metadata()["types"]):
# Get the actual profile for existing_applicability or lookup using profile_id
if existing_applicability:
profile = existing_applicability.profile
else:
unit_profile = UnitProfile.get_collection().find_one({"id": profile_id}, fields=["profile"])
profile = unit_profile["profile"]
call_config = PluginCallConfiguration(plugin_config=profiler_cfg, repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profile, bound_repo_id, call_config, profiler_conduit
)
except NotImplementedError:
logger.debug("Profiler for content type [%s] does not support applicability" % content_type)
return
if existing_applicability:
# Update existing applicability object
existing_applicability.applicability = applicability
existing_applicability.save()
else:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash, bound_repo_id, unit_profile["profile"], applicability
)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def batch_regenerate_applicability(repo_id, existing_applicability_ids):
"""
Regenerate and save applicability data for a batch of existing applicabilities
:param repo_id: Repository id for which applicability is being calculated
:type repo_id: str
:param existing_applicability_ids: Tuple of Object Ids for applicability profiles
:type existing_applicability_ids: tuple of dicts in form of {"_id": ObjectID('mongo-id')}
"""
id_list = [id['_id'] for id in existing_applicability_ids]
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{"_id": {"$in": id_list}})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def regenerate_applicability_for_repos(self, repo_criteria=None):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: pulp.server.db.model.criteria.Criteria
"""
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find({'repo_id':repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id','content_type'])
# Regenerate applicability data for given unit_profile and repo id
self.regenerate_applicability(profile_hash, unit_profile['content_type'],
unit_profile['id'],
repo_id,
existing_applicability)
def remove_orphans():
"""
The RepoProfileApplicability objects can become orphaned over time, as repositories are
deleted, or as consumer profiles change. This method searches for RepoProfileApplicability
objects that reference either repositories or profile hashes that no longer exist in Pulp.
"""
# Find all of the repo_ids that are referenced by RepoProfileApplicability objects
rpa_collection = RepoProfileApplicability.get_collection()
rpa_repo_ids = rpa_collection.distinct('repo_id')
# Find all of the repo_ids that exist in Pulp
repo_ids = Repo.get_collection().distinct('id')
# Find rpa_repo_ids that aren't part of repo_ids
missing_repo_ids = list(set(rpa_repo_ids) - set(repo_ids))
# Remove all RepoProfileApplicability objects that reference these repo_ids
if missing_repo_ids:
rpa_collection.remove({'repo_id': {'$in': missing_repo_ids}})
# Next, we need to find profile_hashes that don't exist in the UnitProfile collection
rpa_profile_hashes = rpa_collection.distinct('profile_hash')
# Find the profile hashes that exist in current UnitProfiles
profile_hashes = UnitProfile.get_collection().distinct('profile_hash')
# Find profile hashes that we have RepoProfileApplicability objects for, but no real
# UnitProfiles
missing_profile_hashes = list(set(rpa_profile_hashes) - set(profile_hashes))
# Remove all RepoProfileApplicability objects that reference these profile hashes
if missing_profile_hashes:
rpa_collection.remove({'profile_hash': {'$in': missing_profile_hashes}})
def consumer_deleted(self, id):
"""
Notification that a consumer has been deleted.
Associated profiles are removed.
@param id: uniquely identifies the consumer.
@type id: str
"""
collection = UnitProfile.get_collection()
for p in self.get_profiles(id):
collection.remove(p, sefe=True)
def consumer_deleted(self, id):
"""
Notification that a consumer has been deleted.
Associated profiles are removed.
@param id: uniquely identifies the consumer.
@type id: str
"""
collection = UnitProfile.get_collection()
for p in self.get_profiles(id):
collection.remove(p, sefe=True)
def test_consumer_unregister_cleanup(self):
# Setup
self.test_create()
# Test
manager = factory.consumer_manager()
manager.unregister(self.CONSUMER_ID)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 0)
def test_consumer_unregister_cleanup(self):
# Setup
self.test_create()
# Test
manager = factory.consumer_manager()
manager.unregister(self.CONSUMER_ID)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({"consumer_id": self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 0)
def delete(self, consumer_id, content_type):
"""
Delete a profile by consumer and content type.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
"""
profile = self.get_profile(consumer_id, content_type)
collection = UnitProfile.get_collection()
collection.remove(profile, safe=True)
def delete(self, consumer_id, content_type):
"""
Delete a profile by consumer and content type.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
"""
profile = self.get_profile(consumer_id, content_type)
collection = UnitProfile.get_collection()
collection.remove(profile, safe=True)
def test_delete(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id':self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 2)
# Test
manager.delete(self.CONSUMER_ID, self.TYPE_1)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id':self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 1)
self.assertEquals(profiles[0]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[0]['content_type'], self.TYPE_2)
self.assertEquals(profiles[0]['profile'], self.PROFILE_2)
def delete(consumer_id, content_type):
"""
Delete a profile by consumer and content type.
:param consumer_id: uniquely identifies the consumer.
:type consumer_id: str
:param content_type: The profile (content) type ID.
:type content_type: str
"""
profile = ProfileManager.get_profile(consumer_id, content_type)
collection = UnitProfile.get_collection()
collection.remove(profile)
def delete(consumer_id, content_type):
"""
Delete a profile by consumer and content type.
:param consumer_id: uniquely identifies the consumer.
:type consumer_id: str
:param content_type: The profile (content) type ID.
:type content_type: str
"""
profile = ProfileManager.get_profile(consumer_id, content_type)
collection = UnitProfile.get_collection()
collection.remove(profile)
def find_by_criteria(criteria):
"""
Return a list of unit profiles that match the provided criteria.
@param criteria: A Criteria object representing a search you want
to perform
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of UnitProfile instances
@rtype: list
"""
return UnitProfile.get_collection().query(criteria)
def find_by_criteria(criteria):
"""
Return a list of unit profiles that match the provided criteria.
@param criteria: A Criteria object representing a search you want
to perform
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of UnitProfile instances
@rtype: list
"""
return UnitProfile.get_collection().query(criteria)
def get_profile(self, consumer_id, content_type):
"""
Get a profile by consumer ID and content type ID.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
@return: The requested profile.
@rtype: object
"""
collection = UnitProfile.get_collection()
query = dict(consumer_id=consumer_id, content_type=content_type)
return collection.find_one(query)
def get_profiles(self, consumer_id):
"""
Get all profiles associated with a consumer.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@return: A list of profiles:
{consumer_id:<str>, content_type:<str>, profile:<dict>}
@rtype: list
"""
collection = UnitProfile.get_collection()
query = dict(consumer_id=consumer_id)
cursor = collection.find(query)
return list(cursor)
def get_profiles(self, consumer_id):
"""
Get all profiles associated with a consumer.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@return: A list of profiles:
{consumer_id:<str>, content_type:<str>, profile:<dict>}
@rtype: list
"""
collection = UnitProfile.get_collection()
query = dict(consumer_id=consumer_id)
cursor = collection.find(query)
return list(cursor)
def test_delete(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 2)
# Test
manager.delete(self.CONSUMER_ID, self.TYPE_1)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 1)
self.assertEquals(profiles[0]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[0]['content_type'], self.TYPE_2)
self.assertEquals(profiles[0]['profile'], self.PROFILE_2)
expected_hash = UnitProfile.calculate_hash(self.PROFILE_2)
self.assertEqual(profiles[0]['profile_hash'], expected_hash)
def test_fetch_by_type2(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
# Test
profile = manager.get_profile(self.CONSUMER_ID, self.TYPE_2)
# Verify
self.assertTrue(profile is not None)
self.assertEquals(profile["consumer_id"], self.CONSUMER_ID)
self.assertEquals(profile["content_type"], self.TYPE_2)
self.assertEquals(profile["profile"], self.PROFILE_2)
def test_delete(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
collection = UnitProfile.get_collection()
cursor = collection.find({"consumer_id": self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 2)
# Test
manager.delete(self.CONSUMER_ID, self.TYPE_1)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({"consumer_id": self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 1)
self.assertEquals(profiles[0]["consumer_id"], self.CONSUMER_ID)
self.assertEquals(profiles[0]["content_type"], self.TYPE_2)
self.assertEquals(profiles[0]["profile"], self.PROFILE_2)
expected_hash = UnitProfile.calculate_hash(self.PROFILE_2)
self.assertEqual(profiles[0]["profile_hash"], expected_hash)
def test_create(self):
# Setup
self.populate()
# Test
manager = factory.consumer_profile_manager()
manager.create(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 1)
self.assertEquals(profiles[0]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[0]['content_type'], self.TYPE_1)
self.assertEquals(profiles[0]['profile'], self.PROFILE_1)
def test_fetch_by_type2(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
# Test
profile = manager.get_profile(self.CONSUMER_ID, self.TYPE_2)
# Verify
self.assertTrue(profile is not None)
self.assertEquals(profile['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profile['content_type'], self.TYPE_2)
self.assertEquals(profile['profile'], self.PROFILE_2)
def test_fetch_by_type1(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
# Test
profile = manager.get_profile(self.CONSUMER_ID, self.TYPE_1)
# Verify
self.assertTrue(profile is not None)
self.assertEquals(profile['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profile['content_type'], self.TYPE_1)
self.assertEquals(profile['profile'], self.PROFILE_1)
expected_hash = UnitProfile.calculate_hash(self.PROFILE_1)
self.assertEqual(profile['profile_hash'], expected_hash)
def test_create(self):
# Setup
self.populate()
# Test
manager = factory.consumer_profile_manager()
manager.create(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
# Verify
collection = UnitProfile.get_collection()
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
profiles = list(cursor)
self.assertEquals(len(profiles), 1)
self.assertEquals(profiles[0]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[0]['content_type'], self.TYPE_1)
self.assertEquals(profiles[0]['profile'], self.PROFILE_1)
expected_hash = UnitProfile.calculate_hash(self.PROFILE_1)
self.assertEqual(profiles[0]['profile_hash'], expected_hash)
def get_profile(self, consumer_id, content_type):
"""
Get a profile by consumer ID and content type ID.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
@return: The requested profile.
@rtype: object
@raise MissingResource when profile not found.
"""
collection = UnitProfile.get_collection()
profile_id = dict(consumer_id=consumer_id, content_type=content_type)
profile = collection.find_one(profile_id)
if profile is None:
raise MissingResource(profile_id=profile_id)
else:
return profile
def get_profile(self, consumer_id, content_type):
"""
Get a profile by consumer ID and content type ID.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
@return: The requested profile.
@rtype: object
@raise MissingResource when profile not found.
"""
collection = UnitProfile.get_collection()
profile_id = dict(consumer_id=consumer_id, content_type=content_type)
profile = collection.find_one(profile_id)
if profile is None:
raise MissingResource(profile_id=profile_id)
else:
return profile
def find_profiles(self, consumer_ids):
"""
Get all profiles associated with given consumers.
@param consumer_ids: A list of consumer IDs.
@type consumer_ids: list
@return: A dict of:
{<consumer_id>:{<content_type>:<profile>}}
@rtype: list
"""
profiles = dict([(c, {}) for c in consumer_ids])
collection = UnitProfile.get_collection()
for p in collection.find({'id':{'$in':profiles.keys()}}):
key = p['consumer_id']
typeid = p['content_type']
profile = p['profile']
entry = profiles[key]
entry[typeid] = profile
return profiles
def find_profiles(self, consumer_ids):
"""
Get all profiles associated with given consumers.
@param consumer_ids: A list of consumer IDs.
@type consumer_ids: list
@return: A dict of:
{<consumer_id>:{<content_type>:<profile>}}
@rtype: list
"""
profiles = dict([(c, {}) for c in consumer_ids])
collection = UnitProfile.get_collection()
for p in collection.find({'id': {'$in': profiles.keys()}}):
key = p['consumer_id']
typeid = p['content_type']
profile = p['profile']
entry = profiles[key]
entry[typeid] = profile
return profiles
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
# the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
# more details.
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({
'repo_id': repo_id
}).batch_size(5)
for existing_applicability in existing_applicabilities:
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def test_multiple_types(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
# Test
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
# Verify
cursor = collection.find({'consumer_id': self.CONSUMER_ID})
cursor.sort('content_type', pymongo.ASCENDING)
profiles = list(cursor)
# Type_1
self.assertEquals(len(profiles), 2)
self.assertEquals(profiles[0]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[0]['content_type'], self.TYPE_1)
self.assertEquals(profiles[0]['profile'], self.PROFILE_1)
self.assertEquals(profiles[1]['consumer_id'], self.CONSUMER_ID)
self.assertEquals(profiles[1]['content_type'], self.TYPE_2)
self.assertEquals(profiles[1]['profile'], self.PROFILE_2)
def test_multiple_types(self):
# Setup
self.populate()
collection = UnitProfile.get_collection()
# Test
manager = factory.consumer_profile_manager()
manager.update(self.CONSUMER_ID, self.TYPE_1, self.PROFILE_1)
manager.update(self.CONSUMER_ID, self.TYPE_2, self.PROFILE_2)
# Verify
cursor = collection.find({"consumer_id": self.CONSUMER_ID})
cursor.sort("content_type", pymongo.ASCENDING)
profiles = list(cursor)
# Type_1
self.assertEquals(len(profiles), 2)
self.assertEquals(profiles[0]["consumer_id"], self.CONSUMER_ID)
self.assertEquals(profiles[0]["content_type"], self.TYPE_1)
self.assertEquals(profiles[0]["profile"], self.PROFILE_1)
self.assertEquals(profiles[1]["consumer_id"], self.CONSUMER_ID)
self.assertEquals(profiles[1]["content_type"], self.TYPE_2)
self.assertEquals(profiles[1]["profile"], self.PROFILE_2)
def update(self, consumer_id, content_type, profile):
"""
Update a unit profile.
Created if not already exists.
@param consumer_id: uniquely identifies the consumer.
@type consumer_id: str
@param content_type: The profile (content) type ID.
@type content_type: str
@param profile: The unit profile
@type profile: object
"""
manager = factory.consumer_manager()
manager.get_consumer(consumer_id)
try:
p = self.get_profile(consumer_id, content_type)
p['profile'] = profile
except MissingResource:
p = UnitProfile(consumer_id, content_type, profile)
collection = UnitProfile.get_collection()
collection.save(p, safe=True)
return p
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def batch_regenerate_applicability(repo_id, profile_hashes):
"""
Regenerate and save applicability data for a batch of existing applicabilities
:param repo_id: Repository id for which applicability is being calculated
:type repo_id: str
:param profile_hashes: Tuple of consumer profile hashes for applicability profiles.
Don't pass too much of these, all the profile data
associated with these hashes is loaded into the memory.
:type profile_hashes: tuple of dicts in form of {'profile_hash': str}
"""
profile_hash_list = [phash['profile_hash'] for phash in profile_hashes]
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({
"repo_id": repo_id,
"profile_hash": {
"$in": profile_hash_list
}
})
for existing_applicability in list(existing_applicabilities):
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'],
repo_id, existing_applicability)
def setUp(self):
super(ProfileManagerTests, self).setUp()
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
def remove_orphans():
"""
The RepoProfileApplicability objects can become orphaned over time, as repositories are
deleted, or as consumer profiles change. This method searches for RepoProfileApplicability
objects that reference either repositories or profile hashes that no longer exist in Pulp.
There is a rare case when some orphaned applicability profiles are not removed:
- a consumer can have multiple profiles and applicability is calculated for a
combination of them
- if only one of the profiles changed, then the applicability for the unchnaged one is
not removed.
- there is no harm, no consequences to the applicability results when requested,
just useless records in the DB
"""
# Find all of the repo_ids that are referenced by RepoProfileApplicability objects
rpa_collection = RepoProfileApplicability.get_collection()
rpa_repo_ids = rpa_collection.distinct('repo_id')
# Find all of the repo_ids that exist in Pulp
repo_ids = model.Repository.objects.distinct('repo_id')
# Find rpa_repo_ids that aren't part of repo_ids
missing_repo_ids = list(set(rpa_repo_ids) - set(repo_ids))
# Remove all RepoProfileApplicability objects that reference these repo_ids
if missing_repo_ids:
rpa_collection.remove({'repo_id': {'$in': missing_repo_ids}})
# The code below has to be compatible with MongoDB 2.6+, it has to workaround
# the 16MB BSON size limit, and no race conditions should be introduced.
# For those reasons it may look complicated or unintuitive, but it does the following:
#
# active_profile_hashes = set(consumer_unit_profile collection)
# profile_hashes_in_applicability = set(repo_profile_applicability collection)
# orphaned_profile_hashes = profile_hashes_in_applicability - active_profile_hashes
# for batch in paginate(orphaned_profile_hashes):
# remove_from_applicability_collection(where profile_hashes in batch)
#
# Find the profile hashes that exist in current UnitProfiles
active_profile_hashes = UnitProfile.get_collection().distinct(
'profile_hash')
# Define a group stage for aggregation to find the profile hashes
# that are present in RepoProfileApplicability collection
group_stage = {
'$group': {
'_id': None,
'rpa_profiles': {
'$addToSet': '$profile_hash'
}
}
}
# Define a project stage to find orphaned profile hashes in the RepoProfileApplicability
project_stage1 = {
"$project": {
"orphaned_profiles": {
"$setDifference": ["$rpa_profiles", active_profile_hashes]
}
}
}
# Unwind the array of results so each element becomes a document itself.
# It's important if results are huge (>16MB)
unwind_stage = {"$unwind": "$orphaned_profiles"}
# Reshape results in a wayi that no indices are violated: _id = profile_hash
project_stage2 = {"$project": {"_id": "$orphaned_profiles"}}
# Write results to a separate collection.
# If a collection exists, old data is substituted with a new one.
out_stage = {"$out": "orphaned_profile_hash"}
# Trigger aggregation pipeline
rpa_collection.aggregate([
group_stage, project_stage1, unwind_stage, project_stage2,
out_stage
],
allowDiskUse=True)
# Remove orphaned applicability profiles using profile hashes from the temporary collection.
# Prepare a list of profiles to remove them in batches in case there are millions of them.
orphaned_profiles_collection = connection.get_collection(
'orphaned_profile_hash')
profiles_batch_size = 100000
profiles_total = orphaned_profiles_collection.count()
_logger.info("Orphaned consumer profiles to process: %s" %
profiles_total)
for skip_idx in range(0, profiles_total, profiles_batch_size):
skip_stage = {"$skip": skip_idx}
limit_stage = {"$limit": profiles_batch_size}
group_stage = {
"$group": {
"_id": None,
"profile_hash": {
"$push": "$_id"
}
}
}
agg_result = orphaned_profiles_collection.aggregate(
[skip_stage, limit_stage, group_stage])
profiles_to_remove = agg_result.next()['profile_hash']
rpa_collection.remove(
{'profile_hash': {
'$in': profiles_to_remove
}})
# Statistics
if profiles_total <= profiles_batch_size + skip_idx:
profiles_removed = profiles_total
else:
profiles_removed = profiles_batch_size + skip_idx
_logger.info("Orphaned consumer profiles processed: %s" %
profiles_removed)
def tearDown(self):
super(ProfileManagerTests, self).tearDown()
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
def regenerate_applicability(profile_hash,
content_type,
profile_id,
bound_repo_id,
existing_applicability=None):
"""
Regenerate and save applicability data for given profile and bound repo id.
If existing_applicability is not None, replace it with the new applicability data.
:param profile_hash: hash of the unit profile
:type profile_hash: basestring
:param content_type: profile (unit) type ID
:type content_type: str
:param profile_id: unique id of the unit profile
:type profile_id: str
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
:param existing_applicability: existing RepoProfileApplicability object to be replaced
:type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given unit_profile
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if (set(repo_content_types) & set(profiler.metadata()['types'])):
# Get the actual profile for existing_applicability or lookup using profile_id
if existing_applicability:
profile = existing_applicability.profile
else:
unit_profile = UnitProfile.get_collection().find_one(
{'id': profile_id}, projection=['profile'])
profile = unit_profile['profile']
call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profile, bound_repo_id, call_config, profiler_conduit)
except NotImplementedError:
msg = "Profiler for content type [%s] does not support applicability" % content_type
_logger.debug(msg)
return
if existing_applicability:
# Update existing applicability object
existing_applicability.applicability = applicability
existing_applicability.save()
else:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash, bound_repo_id, unit_profile['profile'],
applicability)
def tearDown(self):
base.PulpServerTests.tearDown(self)
Consumer.get_collection().remove()
UnitProfile.get_collection().remove()
mock_plugins.reset()