def test_get_bindings(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
binds = conduit.get_bindings(self.CONSUMER_ID)
# Verify
self.assertEquals(1, len(binds))
self.assertTrue(binds[0], self.REPO_ID)
def test_get_units(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
criteria = UnitAssociationCriteria(type_ids=[self.TYPE_1_DEF.id])
units = conduit.get_units(self.REPO_ID, criteria)
# Verify
self.assertEquals(len(units), 9)
def test_get_units(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
criteria = UnitAssociationCriteria(type_ids=[self.TYPE_1_DEF.id])
units = conduit.get_units(self.REPO_ID, criteria)
# Verify
self.assertEquals(len(units), 9)
def test_get_bindings(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
binds = conduit.get_bindings(self.CONSUMER_ID)
# Verify
self.assertEquals(1, len(binds))
self.assertTrue(binds[0], self.REPO_ID)
def test_get_repo_units_additional_field(self):
# Setup
self.populate(additional_key='extra_field')
# Test
conduit = ProfilerConduit()
units = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_1_DEF.id,
additional_unit_fields=['extra_field'])
# Verify that all the units in the repo with given type are returned along with unit_key and extra field
self.assertEquals(len(units), 9)
for u in units:
self.assertTrue('key-1' in u.unit_key)
self.assertTrue('extra_field' in u.metadata)
def test_get_repo_units_additional_field(self):
# Setup
self.populate(additional_key='extra_field')
# Test
conduit = ProfilerConduit()
units = conduit.get_repo_units(self.REPO_ID,
content_type_id=self.TYPE_1_DEF.id,
additional_unit_fields=['extra_field'])
# Verify that all the units in the repo with given type are returned along with unit_key and extra field
self.assertEquals(len(units), 9)
for u in units:
self.assertTrue('key-1' in u.unit_key)
self.assertTrue('extra_field' in u.metadata)
def setUp(self):
self.profiler = wholerepo.WholeRepoProfiler()
self.consumer = Consumer('consumer1', {})
self.units = [{
'type_id': constants.TYPE_PUPPET_MODULE,
'unit_key': {
'name': 'gcc',
'author': 'puppetlabs'
}
}, {
'type_id': constants.TYPE_PUPPET_MODULE,
'unit_key': {
'name': 'stdlib',
'author': 'puppetlabs',
'version': '3.1.1'
}
}, {
'type_id': constants.TYPE_PUPPET_MODULE,
'unit_key': {
'name': 'stdlib',
'author': 'puppetlabs',
'version': '3.2.0'
}
}]
self.conduit = mock.MagicMock(spec=ProfilerConduit())
self.conduit.get_units.return_value = [
AssociatedUnit(constants.TYPE_PUPPET_MODULE, unit['unit_key'], {},
'', '', '', '', '') for unit in self.units
]
def test_get_repo_units(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
units1 = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_1_DEF.id, additional_unit_fields=[])
units2 = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_2_DEF.id, additional_unit_fields=[])
# Verify that all the units in the repo with given type are returned along with unit_key
self.assertEquals(len(units1), 9)
for u in units1:
self.assertTrue('key-1' in u.unit_key)
self.assertFalse('key-2' in u.unit_key)
self.assertEquals(len(units2), 9)
for u in units2:
self.assertTrue('key-2' in u.unit_key)
self.assertFalse('key-1' in u.unit_key)
def test_get_repo_units(self):
# Setup
self.populate()
# Test
conduit = ProfilerConduit()
units1 = conduit.get_repo_units(self.REPO_ID,
content_type_id=self.TYPE_1_DEF.id,
additional_unit_fields=[])
units2 = conduit.get_repo_units(self.REPO_ID,
content_type_id=self.TYPE_2_DEF.id,
additional_unit_fields=[])
# Verify that all the units in the repo with given type are returned along with unit_key
self.assertEquals(len(units1), 9)
for u in units1:
self.assertTrue('key-1' in u.unit_key)
self.assertFalse('key-2' in u.unit_key)
self.assertEquals(len(units2), 9)
for u in units2:
self.assertTrue('key-2' in u.unit_key)
self.assertFalse('key-1' in u.unit_key)
def uninstall_content(consumer_id, units, options):
"""
Uninstall content units on a consumer.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param units: A list of content units to be uninstalled.
:type units: list of:
{ type_id:<str>, type_id:<dict> }
:param options: Uninstall options; based on unit type.
:type options: dict
:return: A task ID that may be used to track the agent request.
:rtype: dict
"""
# track agent operations using a pseudo task
task_id = str(uuid4())
task_tags = [
tags.resource_tag(tags.RESOURCE_CONSUMER_TYPE, consumer_id),
tags.action_tag(tags.ACTION_AGENT_UNIT_UNINSTALL)
]
task = TaskStatus(task_id, 'agent', tags=task_tags).save()
# agent request
manager = managers.consumer_manager()
consumer = manager.get_consumer(consumer_id)
conduit = ProfilerConduit()
collated = Units(units)
for typeid, units in collated.items():
pc = AgentManager._profiled_consumer(consumer_id)
profiler, cfg = AgentManager._profiler(typeid)
units = AgentManager._invoke_plugin(
profiler.uninstall_units,
pc,
units,
options,
cfg,
conduit)
collated[typeid] = units
units = collated.join()
context = Context(consumer, task_id=task_id, consumer_id=consumer_id)
agent = PulpAgent()
agent.content.uninstall(context, units, options)
return task
def uninstall_content(self, consumer_id, units, options):
"""
Uninstall content units on a consumer.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param units: A list of content units to be uninstalled.
:type units: list of:
{ type_id:<str>, type_id:<dict> }
:param options: Uninstall options; based on unit type.
:type options: dict
"""
manager = managers.consumer_manager()
consumer = manager.get_consumer(consumer_id)
conduit = ProfilerConduit()
collated = Units(units)
for typeid, units in collated.items():
pc = self.__profiled_consumer(consumer_id)
profiler, cfg = self.__profiler(typeid)
units = self.__invoke_plugin(profiler.uninstall_units, pc, units,
options, cfg, conduit)
collated[typeid] = units
units = collated.join()
agent = PulpAgent(consumer)
agent.content.uninstall(units, options)
def regenerate_applicability(profile_hash,
content_type,
profile_id,
bound_repo_id,
existing_applicability=None):
"""
Regenerate and save applicability data for given profile and bound repo id.
If existing_applicability is not None, replace it with the new applicability data.
:param profile_hash: hash of the unit profile
:type profile_hash: basestring
:param content_type: profile (unit) type ID
:type content_type: str
:param profile_id: unique id of the unit profile
:type profile_id: str
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
:param existing_applicability: existing RepoProfileApplicability object to be replaced
:type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given unit_profile
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if (set(repo_content_types) & set(profiler.metadata()['types'])):
# Get the actual profile for existing_applicability or lookup using profile_id
if existing_applicability:
profile = existing_applicability.profile
else:
unit_profile = UnitProfile.get_collection().find_one(
{'id': profile_id}, projection=['profile'])
profile = unit_profile['profile']
call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profile, bound_repo_id, call_config, profiler_conduit)
except NotImplementedError:
msg = "Profiler for content type [%s] does not support applicability" % content_type
_logger.debug(msg)
return
if existing_applicability:
# Update existing applicability object
existing_applicability.applicability = applicability
existing_applicability.save()
else:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash, bound_repo_id, unit_profile['profile'],
applicability)
def find_applicable_units(self,
consumer_criteria=None,
repo_criteria=None,
unit_criteria=None,
override_config=None):
"""
Determine and report which of the content units specified by the unit_criteria
are applicable to consumers specified by the consumer_criteria
with repos specified by repo_criteria. If consumer_criteria is None,
all consumers registered to the Pulp server are checked for applicability.
If repo_criteria is None, all repos bound to the consumer are taken
into consideration. If unit_criteria contains an empty list for a specific type,
all units with specific type in the repos bound to the consumer
are taken into consideration.
:param consumer_criteria: The consumer selection criteria.
:type consumer_criteria: dict
:param repo_criteria: The repo selection criteria.
:type repo_criteria: dict
:param unit_criteria: A dictionary of type_id : unit selection criteria
:type units: dict
{<type_id1> : <unit_criteria_for_type_id1>,
<type_id2> : <unit_criteria_for_type_id2>}
:param override_config: Additional configuration options to be accepted from user
:type override_config: dict
:return: applicability reports dictionary keyed by content type id
:rtype: dict
"""
result = {}
conduit = ProfilerConduit()
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
# Process Repo Criteria
if repo_criteria:
# Get repo ids satisfied by specified repo criteria
repo_query_manager = managers.repo_query_manager()
repo_criteria_ids = [
r['id']
for r in repo_query_manager.find_by_criteria(repo_criteria)
]
# if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
if not repo_criteria_ids:
return result
else:
repo_criteria_ids = None
# Process Consumer Criteria
if consumer_criteria:
# Get consumer ids satisfied by specified consumer criteria
consumer_ids = [
c['id'] for c in consumer_query_manager.find_by_criteria(
consumer_criteria)
]
else:
if repo_criteria_ids:
# If repo_criteria is specified, get all the consumers bound to the repos
# satisfied by repo_criteria
bind_criteria = Criteria(
filters={"repo_id": {
"$in": repo_criteria_ids
}})
consumer_ids = [
b['consumer_id']
for b in bind_manager.find_by_criteria(bind_criteria)
]
# Remove duplicate consumer ids
consumer_ids = list(set(consumer_ids))
else:
# Get all consumer ids registered to the Pulp server
consumer_ids = [
c['id'] for c in consumer_query_manager.find_all()
]
# if there are no relevant consumers, return empty result
if not consumer_ids:
return result
else:
# Based on the consumers, get all the repos bound to the consumers in consideration
# and find intersection of repo_criteria_ids and consumer_repo_ids
bind_criteria = Criteria(
filters={"consumer_id": {
"$in": consumer_ids
}})
consumer_repo_ids = [
b['repo_id']
for b in bind_manager.find_by_criteria(bind_criteria)
]
if not repo_criteria_ids:
repo_criteria_ids = list(set(consumer_repo_ids))
else:
repo_criteria_ids = list(
set(consumer_repo_ids) & set(repo_criteria_ids))
if not repo_criteria_ids:
return result
# Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
consumer_profile_and_repo_ids = {}
all_relevant_repo_ids = set()
for consumer_id in consumer_ids:
# Find repos bound to the consumer in consideration and find an intersection of bound repos to the
# repos specified by repo_criteria
consumer_bound_repo_ids = [
b['repo_id']
for b in bind_manager.find_by_consumer(consumer_id)
]
consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
# If repo_criteria is not specified, use repos bound to the consumer, else take intersection
# of repos specified in the criteria and repos bound to the consumer.
if repo_criteria_ids is None:
repo_ids = consumer_bound_repo_ids
else:
repo_ids = list(
set(consumer_bound_repo_ids) & set(repo_criteria_ids))
if repo_ids:
# Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
consumer_profile_and_repo_ids[consumer_id] = {
'repo_ids': repo_ids
}
consumer_profile_and_repo_ids[consumer_id][
'profiled_consumer'] = self.__profiled_consumer(
consumer_id)
if not unit_criteria:
return result
# Call respective profiler api according to the unit type to check for applicability
for unit_type_id, criteria in unit_criteria.items():
# Find a profiler for each type id and find units applicable using that profiler.
profiler, cfg = self.__profiler(unit_type_id)
call_config = PluginCallConfiguration(
plugin_config=cfg,
repo_plugin_config=None,
override_config=override_config)
try:
report_list = profiler.find_applicable_units(
consumer_profile_and_repo_ids, unit_type_id, criteria,
call_config, conduit)
except PulpExecutionException:
report_list = None
if report_list is None:
_LOG.warn(
"Profiler for unit type [%s] is not returning applicability reports"
% unit_type_id)
else:
result[unit_type_id] = report_list
return result
def regenerate_applicability(all_profiles_hash, profiles, bound_repo_id):
"""
Regenerate and save applicability data for given set of profiles and bound repo id.
:param all_profiles_hash: hash of the consumer profiles
:type all_profiles_hash: basestring
:param profiles: profiles data: (profile_hash, content_type, profile_id)
:type profiles: list of tuples
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given profiles.
# The assumption is that the same profiler is used for all the content types, so different
# profilers are not supported at the moment.
# Take the content type from the first profile.
content_type = profiles[0][1]
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if (set(repo_content_types) & set(profiler.metadata()['types'])):
profile_ids = [p_id for _, _, p_id in profiles]
unit_profiles = UnitProfile.get_collection().find(
{'id': {
'$in': profile_ids
}},
projection=['profile', 'content_type', 'profile_hash'])
try:
profiles = [(p['profile_hash'], p['content_type'],
p['profile']) for p in unit_profiles]
except TypeError:
# It means that p = None.
# Consumer can be removed during applicability regeneration,
# so it is possible that its profile no longer exists. It is harmless.
return
call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profiles, bound_repo_id, call_config, profiler_conduit)
except NotImplementedError:
msg = "Profiler for content type [%s] does not support applicability" % content_type
_logger.debug(msg)
return
# Save applicability results on each of the profiles. The results are duplicated.
# It's a compromise to have applicability data available in any applicability profile
# record in the DB.
for profile in profiles:
profile_hash = profile[0]
try:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash=profile_hash,
repo_id=bound_repo_id,
# profiles can be large, the one in
# repo_profile_applicability collection
# is no longer used,
# it's a duplicated data
# from the consumer_unit_profiles
# collection.
profile=[],
applicability=applicability,
all_profiles_hash=all_profiles_hash)
except DuplicateKeyError:
applicability_dict = RepoProfileApplicability.get_collection(
).find_one({
'repo_id': bound_repo_id,
'all_profiles_hash': all_profiles_hash,
'profile_hash': profile_hash
})
existing_applicability = RepoProfileApplicability(
**applicability_dict)
existing_applicability.applicability = applicability
existing_applicability.save()