def test_find_by_criteria(self, mock_repo_qs):
self.populate()
manager = factory.consumer_bind_manager()
manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID,
self.NOTIFY_AGENT, self.BINDING_CONFIG)
# Test
criteria = Criteria({'consumer_id': self.CONSUMER_ID})
bindings = manager.find_by_criteria(criteria)
bind = bindings[0]
self.assertEqual(len(bindings), 1)
self.assertEqual(bind['consumer_id'], self.CONSUMER_ID)
self.assertEqual(bind['repo_id'], self.REPO_ID)
self.assertEqual(bind['distributor_id'], self.DISTRIBUTOR_ID)
self.assertEqual(bind['notify_agent'], self.NOTIFY_AGENT)
self.assertEqual(bind['binding_config'], self.BINDING_CONFIG)
# Test ($in)
criteria = Criteria({'consumer_id': {'$in': [self.CONSUMER_ID]}})
bindings = manager.find_by_criteria(criteria)
bind = bindings[0]
self.assertEqual(len(bindings), 1)
self.assertEqual(bind['consumer_id'], self.CONSUMER_ID)
self.assertEqual(bind['repo_id'], self.REPO_ID)
self.assertEqual(bind['distributor_id'], self.DISTRIBUTOR_ID)
self.assertEqual(bind['notify_agent'], self.NOTIFY_AGENT)
self.assertEqual(bind['binding_config'], self.BINDING_CONFIG)
def post(self, request, consumer_id):
"""
Creates an async task to regenerate content applicability data for given consumer.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param consumer_id: The consumer ID.
:type consumer_id: str
:raises MissingResource: if some parameters are missing
:raises OperationPostponed: when an async operation is performed.
"""
consumer_query_manager = factory.consumer_query_manager()
if consumer_query_manager.find_by_id(consumer_id) is None:
raise MissingResource(consumer_id=consumer_id)
consumer_criteria = Criteria(filters={'consumer_id': consumer_id})
task_tags = [tags.action_tag('consumer_content_applicability_regeneration')]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_CONSUMER_TYPE,
consumer_id,
(consumer_criteria.as_dict(),),
tags=task_tags)
raise OperationPostponed(async_result)
def post(self, request, consumer_id):
"""
Creates an async task to regenerate content applicability data for given consumer.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param consumer_id: The consumer ID.
:type consumer_id: str
:raises MissingResource: if some parameters are missing
:raises OperationPostponed: when an async operation is performed.
"""
consumer_query_manager = factory.consumer_query_manager()
if consumer_query_manager.find_by_id(consumer_id) is None:
raise MissingResource(consumer_id=consumer_id)
consumer_criteria = Criteria(filters={'consumer_id': consumer_id})
task_tags = [
tags.action_tag('consumer_content_applicability_regeneration')
]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_CONSUMER_TYPE,
consumer_id, (consumer_criteria.as_dict(), ),
tags=task_tags)
raise OperationPostponed(async_result)
class RepoGroupAssociationTests(base.PulpWebserviceTests):
def setUp(self):
super(RepoGroupAssociationTests, self).setUp()
self.manager = manager_factory.repo_group_manager()
def clean(self):
super(RepoGroupAssociationTests, self).clean()
RepoGroup.get_collection().remove()
@mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
@mock.patch(
'pulp.server.managers.repo.group.cud.RepoGroupManager.associate')
def test_associate(self, mock_associate, mock_from_client):
# the CallRequest stuff made mocking out a repo group very difficult,
# so I punted on that.
self.manager.create_repo_group('rg1')
post_data = {'criteria': {'filters': {'id': {'$in': ['repo1']}}}}
status, body = self.post('/v2/repo_groups/rg1/actions/associate/',
post_data)
self.assertEqual(status, 200)
self.assertEqual(mock_associate.call_count, 1)
call_args = mock_associate.call_args[0]
self.assertEqual(call_args[0], 'rg1')
# verify that it created and used a Criteria instance
self.assertEqual(call_args[1], mock_from_client.return_value)
self.assertEqual(mock_from_client.call_args[0][0],
{'filters': {
'id': {
'$in': ['repo1']
}
}})
@mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
@mock.patch(
'pulp.server.managers.repo.group.cud.RepoGroupManager.unassociate')
def test_unassociate(self, mock_unassociate, mock_from_client):
# the CallRequest stuff made mocking out a repo group very difficult,
# so I punted on that.
self.manager.create_repo_group('rg1')
post_data = {'criteria': {'filters': {'id': {'$in': ['repo1']}}}}
status, body = self.post('/v2/repo_groups/rg1/actions/unassociate/',
post_data)
self.assertEqual(status, 200)
self.assertEqual(mock_unassociate.call_count, 1)
call_args = mock_unassociate.call_args[0]
self.assertEqual(call_args[0], 'rg1')
# verify that it created and used a Criteria instance
self.assertEqual(call_args[1], mock_from_client.return_value)
self.assertEqual(mock_from_client.call_args[0][0],
{'filters': {
'id': {
'$in': ['repo1']
}
}})
def POST(self):
"""
Determine content applicability.
body {
consumer_criteria:<dict> or None,
repo_criteria:<dict> or None,
unit_criteria: <dict of type_id : unit_criteria> or None,
override_config: <dict> or None
}
:return:
When report_style is 'by_consumer' -
A dict of applicability reports keyed by consumer ID.
Each consumer report is:
{ <unit_type_id1> : [<ApplicabilityReport>],
<unit_type_id1> : [<ApplicabilityReport>]},
}
When report_style is 'by_units' -
A dict of <unit_type_id1>: [<ApplicabilityReport>]
where applicability_report.summary contains a list of applicable consumer ids.
:rtype: dict
"""
body = self.params()
consumer_criteria = body.get('consumer_criteria', None)
repo_criteria = body.get('repo_criteria', None)
units = body.get('unit_criteria', None)
override_config = body.get('override_config', None)
if consumer_criteria:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
if repo_criteria:
repo_criteria = Criteria.from_client_input(repo_criteria)
unit_criteria = {}
if units:
for type_id, criteria in units.items():
# process if criteria is not empty
if criteria:
unit_criteria[type_id] = Criteria.from_client_input(criteria)
else:
unit_criteria[type_id] = criteria
manager = managers.consumer_applicability_manager()
report = manager.find_applicable_units(consumer_criteria, repo_criteria, unit_criteria, override_config)
for unit_type_id, applicability_reports in report.items():
if isinstance(applicability_reports, list):
report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in applicability_reports]
else:
for consumer_id, report_list in applicability_reports.items():
report[unit_type_id][consumer_id] = [serialization.consumer.applicability_report(r) for r in report_list]
return self.ok(report)
class ConsumerGroupAssociationTests(base.PulpWebserviceTests):
def setUp(self):
super(ConsumerGroupAssociationTests, self).setUp()
self.manager = managers.consumer_group_manager()
def clean(self):
super(ConsumerGroupAssociationTests, self).clean()
ConsumerGroup.get_collection().remove()
@mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
@mock.patch(
'pulp.server.managers.consumer.group.cud.ConsumerGroupManager.associate'
)
def test_associate(self, mock_associate, mock_from_client):
self.manager.create_consumer_group('cg1')
post_data = {'criteria': {'filters': {'id': {'$in': ['consumer1']}}}}
status, body = self.post('/v2/consumer_groups/cg1/actions/associate/',
post_data)
self.assertEqual(status, 200)
self.assertEqual(mock_associate.call_count, 1)
call_args = mock_associate.call_args[0]
self.assertEqual(call_args[0], 'cg1')
# verify that it created and used a Criteria instance
self.assertEqual(call_args[1], mock_from_client.return_value)
self.assertEqual(mock_from_client.call_args[0][0],
{'filters': {
'id': {
'$in': ['consumer1']
}
}})
@mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
@mock.patch(
'pulp.server.managers.consumer.group.cud.ConsumerGroupManager.unassociate'
)
def test_unassociate(self, mock_unassociate, mock_from_client):
self.manager.create_consumer_group('cg1')
post_data = {'criteria': {'filters': {'id': {'$in': ['consumer1']}}}}
status, body = self.post(
'/v2/consumer_groups/cg1/actions/unassociate/', post_data)
self.assertEqual(status, 200)
self.assertEqual(mock_unassociate.call_count, 1)
call_args = mock_unassociate.call_args[0]
self.assertEqual(call_args[0], 'cg1')
# verify that it created and used a Criteria instance
self.assertEqual(call_args[1], mock_from_client.return_value)
self.assertEqual(mock_from_client.call_args[0][0],
{'filters': {
'id': {
'$in': ['consumer1']
}
}})
def __populate_units(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and populate units for each type
if the criteria is empty.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding units. If unit_ids_by_type dict
is empty, return unit ids corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all unit ids
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager(
)
result_units = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = Criteria(filters={
"repo_id": {
"$in": repo_ids
},
"unit_type_id": unit_type_id
},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(
criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id, []).extend(
list(set(pulp_unit_ids)))
else:
result_units.setdefault(unit_type_id, []).extend(unit_ids)
else:
# If units are not specified, consider all units in given repos.
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = Criteria(filters={
"repo_id": {
"$in": repo_ids
},
"unit_type_id": unit_type_id
},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(
criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id,
[]).extend(list(set(pulp_unit_ids)))
return result_units
def check_workers(self):
"""
Look for missing workers, and dispatch a cleanup task if one goes missing.
To find a missing worker, filter the Workers model for entries older than
utcnow() - WORKER_TIMEOUT_SECONDS. The heartbeat times are stored in native UTC, so this is
a comparable datetime.
For each missing worker found, dispatch a _delete_worker task requesting that the resource
manager delete the Worker and cleanup any associated work.
This method logs and the debug and error levels.
"""
msg = _('Looking for workers missing for more than %s seconds'
) % self.WORKER_TIMEOUT_SECONDS
_logger.debug(msg)
oldest_heartbeat_time = datetime.utcnow() - timedelta(
seconds=self.WORKER_TIMEOUT_SECONDS)
worker_criteria = Criteria(
filters={'last_heartbeat': {
'$lt': oldest_heartbeat_time
}},
fields=('_id', 'last_heartbeat', 'num_reservations'))
worker_list = list(resources.filter_workers(worker_criteria))
for worker in worker_list:
msg = _(
"Workers '%s' has gone missing, removing from list of workers"
) % worker.name
_logger.error(msg)
_delete_worker.apply_async(args=(worker.name, ),
queue=RESOURCE_MANAGER_QUEUE)
def post(self, request):
"""
Dispatch a task to regenerate content applicability data for repositories that match
the criteria passed in the body.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:raises exceptions.MissingValue: if repo_critera is not a body parameter
:raises exceptions.InvalidValue: if repo_critera (dict) has unsupported keys,
the manager will raise an InvalidValue for the
specific keys. Here, we create a parent exception
for repo_criteria and include the specific keys as
child exceptions.
:raises exceptions.OperationPostponed: dispatch a task
"""
class GroupCallReport(dict):
def serialize(self):
return self
repo_criteria_body = request.body_as_json.get('repo_criteria', None)
parallel = request.body_as_json.get('parallel', False)
if repo_criteria_body is None:
raise exceptions.MissingValue('repo_criteria')
try:
repo_criteria = Criteria.from_client_input(repo_criteria_body)
except exceptions.InvalidValue, e:
invalid_criteria = exceptions.InvalidValue('repo_criteria')
invalid_criteria.add_child_exception(e)
raise invalid_criteria
def post(self, request):
"""
Dispatch a task to regenerate content applicability data for repositories that match
the criteria passed in the body.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:raises exceptions.MissingValue: if repo_critera is not a body parameter
:raises exceptions.InvalidValue: if repo_critera (dict) has unsupported keys,
the manager will raise an InvalidValue for the
specific keys. Here, we create a parent exception
for repo_criteria and include the specific keys as
child exceptions.
:raises exceptions.OperationPostponed: dispatch a task
"""
class GroupCallReport(dict):
def serialize(self):
return self
repo_criteria_body = request.body_as_json.get('repo_criteria', None)
if repo_criteria_body is None:
raise exceptions.MissingValue('repo_criteria')
try:
repo_criteria = Criteria.from_client_input(repo_criteria_body)
except exceptions.InvalidValue, e:
invalid_criteria = exceptions.InvalidValue('repo_criteria')
invalid_criteria.add_child_exception(e)
raise invalid_criteria
def get_workers():
"""
:returns: list of workers with their heartbeats
:rtype: list
"""
empty_criteria = Criteria()
return resources.filter_workers(empty_criteria)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def get(self, request, type_id):
"""
Return a response with a serialized list of the content units of the specified type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: the list of content units will be limited to this type
:type type_id: str
:return: response with a serialized list of dicts, one for each unit of the type.
:rtype: django.http.HttpResponse
"""
cqm = factory.content_query_manager()
all_units = cqm.find_by_criteria(type_id, Criteria())
all_processed_units = []
for unit in all_units:
unit = serialization.content.content_unit_obj(unit)
unit.update({
'_href':
'/'.join(
[request.get_full_path().rstrip('/'), unit['_id'], ''])
})
unit.update({
'children':
serialization.content.content_unit_child_link_objs(unit)
})
all_processed_units.append(unit)
return generate_json_response_with_pulp_encoder(all_processed_units)
def _get_repo_consumer_map(consumer_ids=[], repo_ids=[]):
"""
Create a repo-consumer map:
{repo_id: [consumer_id1, consumer_id2,...]
Data can be limited to specific consumers or to specific repos.
:param consumer_ids: consumers which should be included
:type consumer_ids: list
:param repo_ids: repositories which should be included
:type repo_ids: list
:return: repo-consumer map described above
:rtype: dict
"""
bind_manager = managers.consumer_bind_manager()
filters = {}
if consumer_ids:
filters = {'consumer_id': {'$in': consumer_ids}}
elif repo_ids:
filters = {'repo_id': {'$in': repo_ids}}
bind_criteria = Criteria(filters=filters,
fields=['repo_id', 'consumer_id'])
all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)
repo_consumers_map = {}
for binding in all_repo_bindings:
repo_consumers_map.setdefault(binding['repo_id'],
[]).append(binding['consumer_id'])
return repo_consumers_map
def test_find_by_criteria_with_result(self):
tags = ['test', 'tags']
TaskStatus(task_id='1').save()
TaskStatus(task_id='2', tags=tags).save()
result = 'done'
TaskStatus(task_id='3',
tags=tags,
state=constants.CALL_FINISHED_STATE,
result=result).save()
filters = {'tags': tags, 'task_id': {'$in': ['1', '3']}}
fields = ['task_id', 'tags', 'result']
limit = 1
sort = (('task_id', DESCENDING), )
criteria = Criteria(filters=filters,
fields=fields,
limit=limit,
sort=sort)
query_set = TaskStatus.objects.find_by_criteria(criteria)
self.assertEqual(len(query_set), 1)
self.assertEqual(query_set[0].task_id, '3')
self.assertEqual(query_set[0].result, result)
task_state_default = constants.CALL_WAITING_STATE
self.assertEqual(query_set[0].state, task_state_default)
def GET(self, type_id):
"""
List all the available content units.
"""
cqm = factory.content_query_manager()
units = cqm.find_by_criteria(type_id, Criteria())
return self.ok([self.process_unit(unit) for unit in units])
def translate_criteria(self, model, crit):
"""
To preserve backwards compatability of our search API, we must translate the fields from
the external representation to the internal representation. This is done most often with
'id' since this is not an allowable key in the database in Mongoengine.
This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
be a dictionary containing the fields that have been renamed in the format:
{'internal repr': 'external repr'}
:param model: the class that defines this document's fields
:type model: sublcass of mongoengine.Document
:param crit: criteria object to be translated from external to internal representation
:type crit: pulp.server.db.model.criteria.Criteria
:return: translated Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
# Circular import avoidance, since criteria imports models which imports serializers
from pulp.server.db.model.criteria import Criteria
crit_dict = crit.as_dict()
if crit.filters:
crit_dict['filters'] = self.translate_filters(model, crit.filters)
if crit.sort:
sort = [(self.translate_field(model, field), direc)
for field, direc in crit.sort]
crit_dict['sort'] = sort
if crit.fields:
crit_dict['fields'] = [
self.translate_field(model, field) for field in crit.fields
]
return Criteria.from_dict(crit_dict)
def POST(self):
"""
Creates an async task to regenerate content applicability data for given updated
repositories.
body {repo_criteria:<dict>}
"""
body = self.params()
repo_criteria = body.get('repo_criteria', None)
if repo_criteria is None:
raise exceptions.MissingValue('repo_criteria')
try:
repo_criteria = Criteria.from_client_input(repo_criteria)
except:
raise exceptions.InvalidValue('repo_criteria')
manager = manager_factory.applicability_regeneration_manager()
regeneration_tag = action_tag('applicability_regeneration')
call_request = CallRequest(manager.regenerate_applicability_for_repos,
[repo_criteria],
tags = [regeneration_tag])
# allow only one applicability regeneration task at a time
call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE,
dispatch_constants.RESOURCE_ANY_ID)
return execution.execute_async(self, call_request)
def _get_total(self, id_list=None, ignore_filter=False):
"""
Return the total number of units that are processed by this step.
This is used generally for progress reporting. The value returned should not change
during the processing of the step.
:param id_list: List of type ids to get the total count of
:type id_list: list of str
:param ignore_filter: Ignore the association filter and get all units of the given types
:type ignore_filter: bool
"""
if id_list is None:
id_list = self.unit_type
total = 0
types_to_query = set(id_list).difference(self.skip_list)
if not ignore_filter and self.association_filters:
# We are copying using a filter so we have to get everything
new_filter = copy.deepcopy(self.association_filters)
new_filter['unit_type_id'] = {'$in': list(types_to_query)}
criteria = Criteria(filters=new_filter)
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
units_cursor = association_query_manager.find_by_criteria(criteria)
total = units_cursor.count()
else:
for type_id in types_to_query:
total += self.parent.repo.content_unit_counts.get(type_id, 0)
return total
def handle_worker_heartbeat(event):
"""
Celery event handler for 'worker-heartbeat' events.
The event is first parsed and logged. If this event is from the resource manager, there is
no further processing to be done. Then the existing Worker objects are searched
for one to update. If an existing one is found, it is updated. Otherwise a new
Worker entry is created. Logging at the info and debug level is also done.
:param event: A celery event to handle.
:type event: dict
"""
event_info = _parse_and_log_event(event)
# if this is the resource_manager do nothing
if _is_resource_manager(event):
return
find_worker_criteria = Criteria(filters={'_id': event_info['worker_name']},
fields=('_id', 'last_heartbeat',
'num_reservations'))
find_worker_list = list(resources.filter_workers(find_worker_criteria))
if find_worker_list:
Worker.get_collection().find_and_modify(
query={'_id': event_info['worker_name']},
update={'$set': {
'last_heartbeat': event_info['timestamp']
}})
else:
new_worker = Worker(event_info['worker_name'], event_info['timestamp'])
msg = _("New worker '%(worker_name)s' discovered") % event_info
_logger.info(msg)
new_worker.save()
def expand_consumers(options, consumers):
"""
Expand a list of users based on flags specified in the
post body or query parameters. The _href is always added by the
serialization function used.
Supported options:
details - synonym for: (bindings=True,)
bindings - include bindings
@param options: The (expanding) options.
@type options: dict
@param consumers: A list of consumers
@type consumers: list
@return: A list of expanded consumers.
@rtype: list
"""
if options.get('details', False):
options['bindings'] = True
# add bindings
if options.get('bindings', False):
ids = [c['id'] for c in consumers]
manager = managers.consumer_bind_manager()
criteria = Criteria({'consumer_id': {'$in': ids}})
bindings = manager.find_by_criteria(criteria)
collated = {}
for b in bindings:
lst = collated.setdefault(b['consumer_id'], [])
lst.append(b)
for _consumer in consumers:
_consumer['bindings'] = \
[serialization.binding.serialize(b, False)
for b in collated.get(_consumer['id'], [])]
return consumers
def POST(self, repo_group_id):
criteria = Criteria.from_client_input(self.params().get('criteria', {}))
manager = managers_factory.repo_group_manager()
manager.unassociate(repo_group_id, criteria)
collection = RepoGroup.get_collection()
group = collection.find_one({'id': repo_group_id})
return self.ok(group['repo_ids'])
def _get_query_results_from_get(self, ignore_fields=None):
"""
Looks for query parameters that define a Criteria, and returns the
results of a search based on that Criteria.
@param ignore_fields: Field names to ignore. All other fields will be
used in an attempt to generate a Criteria
instance, which will fail if unexpected field
names are present.
@type ignore_fields: list
@return: list of documents from the DB that match the given criteria
for the collection associated with this controller
@rtype: list
"""
input = web.input(field=[])
if ignore_fields:
for field in ignore_fields:
input.pop(field, None)
# rename this to 'fields' within the dict, and omit it if empty so we
# default to getting all fields
fields = input.pop('field')
if fields:
if 'id' not in fields and u'id' not in fields:
fields.append('id')
input['fields'] = fields
criteria = Criteria.from_client_input(input)
return list(self.query_method(criteria))
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async(
(repo_id, batch), **{'group_id': task_group_id})
return task_group_id
def post(self, request):
"""
Creates an async task to regenerate content applicability data for given consumers.
body {consumer_criteria:<dict>}
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:raises MissingValue: if some parameters are missing
:raises InvalidValue: if some parameters are invalid
:raises OperationPostponed: when an async operation is performed.
"""
body = request.body_as_json
consumer_criteria = body.get('consumer_criteria', None)
if consumer_criteria is None:
raise MissingValue('consumer_criteria')
try:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
except:
raise InvalidValue('consumer_criteria')
task_tags = [tags.action_tag('content_applicability_regeneration')]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE,
tags.RESOURCE_ANY_ID, (consumer_criteria.as_dict(), ),
tags=task_tags)
raise OperationPostponed(async_result)
def test_profiler_no_exception(self):
# Setup
self.populate()
profiler, cfg = plugins.get_profiler_by_type('rpm')
profiler.find_applicable_units = Mock(side_effect=KeyError)
# Test
user_specified_unit_criteria = {
'rpm': {
"filters": {
"name": {
"$in": ['zsh', 'ksh']
}
}
},
'mock-type': {
"filters": {
"name": {
"$in": ['abc', 'def']
}
}
}
}
unit_criteria = {}
for type_id, criteria in user_specified_unit_criteria.items():
unit_criteria[type_id] = Criteria.from_client_input(criteria)
manager = factory.consumer_applicability_manager()
result = manager.find_applicable_units(self.CONSUMER_CRITERIA,
self.REPO_CRITERIA,
unit_criteria)
self.assertTrue(result == {})
def test_no_exception_for_profiler_notfound(self):
# Setup
self.populate()
# Test
user_specified_unit_criteria = {
'rpm': {
"filters": {
"name": {
"$in": ['zsh']
}
}
},
'xxx': {
"filters": {
"name": {
"$in": ['abc']
}
}
}
}
unit_criteria = {}
for type_id, criteria in user_specified_unit_criteria.items():
unit_criteria[type_id] = Criteria.from_client_input(criteria)
manager = factory.consumer_applicability_manager()
result = manager.find_applicable_units(self.CONSUMER_CRITERIA,
self.REPO_CRITERIA,
unit_criteria)
self.assertTrue(result == {})
def translate_criteria(self, model, crit):
"""
To preserve backwards compatability of our search API, we must translate the fields from
the external representation to the internal representation. This is done most often with
'id' since this is not an allowable key in the database in Mongoengine.
This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
be a dictionary containing the fields that have been renamed in the format:
{'internal repr': 'external repr'}
:param model: the class that defines this document's fields
:type model: sublcass of mongoengine.Document
:param crit: criteria object to be translated from external to internal representation
:type crit: pulp.server.db.model.criteria.Criteria
:return: translated Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
# Circular import avoidance, since criteria imports models which imports serializers
from pulp.server.db.model.criteria import Criteria
crit_dict = crit.as_dict()
if crit.filters:
crit_dict['filters'] = self._translate_filters(model, crit.filters)
if crit.sort:
sort = [(self._translate(model, field), direc) for field, direc in crit.sort]
crit_dict['sort'] = sort
if crit.fields:
crit_dict['fields'] = [self._translate(model, field) for field in crit.fields]
return Criteria.from_dict(crit_dict)
def POST(self, consumer_group_id):
criteria = Criteria.from_client_input(self.params().get('criteria', {}))
manager = managers_factory.consumer_group_manager()
manager.unassociate(consumer_group_id, criteria)
query_manager = managers_factory.consumer_group_query_manager()
group = query_manager.get_group(consumer_group_id)
return self.ok(group['consumer_ids'])
def post(self, request):
"""
Creates an async task to regenerate content applicability data for given consumers.
body {consumer_criteria:<dict>}
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:raises MissingValue: if some parameters are missing
:raises InvalidValue: if some parameters are invalid
:raises OperationPostponed: when an async operation is performed.
"""
body = request.body_as_json
consumer_criteria = body.get('consumer_criteria', None)
if consumer_criteria is None:
raise MissingValue('consumer_criteria')
try:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
except:
raise InvalidValue('consumer_criteria')
task_tags = [tags.action_tag('content_applicability_regeneration')]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
(consumer_criteria.as_dict(),), tags=task_tags)
raise OperationPostponed(async_result)
def test_filter(self):
"""
Test a filter operation to make sure the results appear to be correct.
"""
# Make three workers. We'll filter for two of them.
now = datetime.utcnow()
kw_1 = Worker('worker_1', now)
kw_1.save()
kw_2 = Worker('worker_2', now)
kw_2.save()
kw_3 = Worker('worker_3', now)
kw_3.save()
criteria = Criteria(filters={'_id': {
'$gt': 'worker_1'
}},
sort=[('_id', pymongo.ASCENDING)])
workers = resources.filter_workers(criteria)
# Let's assert that workers is a generator, and then let's cast it to a list so it's easier
# to test that we got the correct instances back.
self.assertEqual(type(workers), types.GeneratorType)
workers = list(workers)
self.assertEqual(all([isinstance(w, Worker) for w in workers]), True)
self.assertEqual(workers[0].name, 'worker_2')
self.assertEqual(workers[1].name, 'worker_3')
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ["id"]
repo_ids = [r["id"] for r in repo_query_manager.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection().find({"repo_id": repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability["profile_hash"]
unit_profile = UnitProfile.get_collection().find_one(
{"profile_hash": profile_hash}, fields=["id", "content_type"]
)
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile["content_type"], unit_profile["id"], repo_id, existing_applicability
)
def _add_repo_memberships(units, type_id):
"""
For a list of units, find what repos each is a member of, and add a list
of repo_ids to each unit.
:param units: list of unit documents
:type units: list of dicts
:param type_id: content type id
:type type_id: str
:return: same list of units that was passed in, only for convenience.
units are modified in-place
"""
# quick return if there is nothing to do
if not units:
return units
unit_ids = [unit['_id'] for unit in units]
criteria = Criteria(
filters={'unit_id': {'$in': unit_ids}, 'unit_type_id': type_id},
fields=('repo_id', 'unit_id')
)
associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria)
unit_ids = None
criteria = None
association_map = {}
for association in associations:
association_map.setdefault(association['unit_id'], set()).add(
association['repo_id'])
for unit in units:
unit['repository_memberships'] = list(association_map.get(unit['_id'], []))
return units
def POST(self, consumer_group_id):
criteria = Criteria.from_client_input(self.params().get(
'criteria', {}))
manager = managers_factory.consumer_group_manager()
manager.unassociate(consumer_group_id, criteria)
query_manager = managers_factory.consumer_group_query_manager()
group = query_manager.get_group(consumer_group_id)
return self.ok(group['consumer_ids'])
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete. In the database, the _id
field is the name.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': name}
logger.error(msg)
# Delete the worker document
worker_list = list(
resources.filter_workers(Criteria(filters={'_id': name})))
if len(worker_list) > 0:
worker_document = worker_list[0]
worker_document.delete()
# Delete all reserved_resource documents for the worker
ReservedResource.get_collection().remove({'worker_name': name})
# Cancel all of the tasks that were assigned to this worker's queue
worker = Worker.from_bson({'_id': name})
for task in TaskStatusManager.find_by_criteria(
Criteria(
filters={
'worker_name': worker.name,
'state': {
'$in': constants.CALL_INCOMPLETE_STATES
}
})):
cancel(task['task_id'])
def populate(self):
manager = managers.consumer_manager()
for consumer_id in CONSUMER_IDS:
manager.register(consumer_id)
manager = managers.consumer_group_manager()
manager.create_consumer_group(GROUP_ID)
for consumer_id in CONSUMER_IDS:
criteria = Criteria(filters={'id': consumer_id}, fields=['id'])
manager.associate(GROUP_ID, criteria)
def get_enabled():
"""
Get schedules that are enabled, that is, their "enabled" attribute is True
:return: pymongo cursor of ScheduledCall database objects
:rtype: pymongo.cursor.Cursor
"""
criteria = Criteria(filters={'enabled': True})
return ScheduledCall.get_collection().query(criteria)
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database. This Task can only safely be
performed by the resource manager at this time, so be sure to queue it in the
RESOURCE_MANAGER_QUEUE.
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the work did not shut down normally.
:param name: The name of the worker you wish to delete. In the database, the _id
field is the name.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
worker_list = list(
resources.filter_workers(Criteria(filters={'_id': name})))
if len(worker_list) == 0:
# Potentially _delete_worker() may be called with the database not containing any entries.
# https://bugzilla.redhat.com/show_bug.cgi?id=1091922
return
worker = worker_list[0]
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': worker.name}
logger.error(msg)
# Cancel all of the tasks that were assigned to this worker's queue
for task in TaskStatusManager.find_by_criteria(
Criteria(
filters={
'queue': worker.queue_name,
'state': {
'$in': constants.CALL_INCOMPLETE_STATES
}
})):
cancel(task['task_id'])
# Finally, delete the worker
worker.delete()
def POST(self, consumer_id):
"""
Creates an async task to regenerate content applicability data for given consumer.
:param consumer_id: The consumer ID.
:type consumer_id: basestring
"""
consumer_query_manager = managers.consumer_query_manager()
if consumer_query_manager.find_by_id(consumer_id) is None:
raise MissingResource(consumer_id=consumer_id)
consumer_criteria = Criteria(filters={'consumer_id': consumer_id})
task_tags = [tags.action_tag('consumer_content_applicability_regeneration')]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_CONSUMER_TYPE,
consumer_id,
(consumer_criteria.as_dict(),),
tags=task_tags)
raise OperationPostponed(async_result)
def POST(self, consumer_group_id):
criteria = Criteria.from_client_input(self.params().get('criteria', {}))
manager = managers_factory.consumer_group_manager()
tags = [resource_tag(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id),
action_tag('consumer_group_unassociate')]
call_request = CallRequest(manager.unassociate,
[consumer_group_id, criteria],
tags=tags)
call_request.updates_resource(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id)
matched = execution.execute(call_request)
return self.ok(matched)
def test_no_exception_for_profiler_notfound(self):
# Setup
self.populate()
# Test
user_specified_unit_criteria = {'rpm': {"filters": {"name": {"$in":['zsh']}}},
'xxx': {"filters": {"name": {"$in":['abc']}}}
}
unit_criteria = {}
for type_id, criteria in user_specified_unit_criteria.items():
unit_criteria[type_id] = Criteria.from_client_input(criteria)
manager = factory.consumer_applicability_manager()
result = manager.find_applicable_units(self.CONSUMER_CRITERIA, self.REPO_CRITERIA, unit_criteria)
self.assertTrue(result == {})
def POST(self, consumer_group_id):
criteria = Criteria.from_client_input(self.params().get('criteria', {}))
manager = managers_factory.consumer_group_manager()
tags = [resource_tag(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id),
action_tag('consumer_group_associate')]
call_request = CallRequest(manager.associate,
[consumer_group_id, criteria],
tags=tags)
call_request.updates_resource(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id)
execution.execute(call_request)
collection = ConsumerGroup.get_collection()
group = collection.find_one({'id': consumer_group_id})
return self.ok(group['consumer_ids'])
def POST(self, repo_group_id):
criteria = Criteria.from_client_input(self.params().get('criteria', {}))
manager = managers_factory.repo_group_manager()
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_GROUP_TYPE, repo_group_id),
action_tag('repo_group_unassociate')]
call_request = CallRequest(manager.unassociate,
[repo_group_id, criteria],
tags=tags)
call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_GROUP_TYPE, repo_group_id)
execution.execute(call_request)
collection = RepoGroup.get_collection()
group = collection.find_one({'id': repo_group_id})
return self.ok(group['repo_ids'])
def GET(self):
valid_filters = ['tag']
filters = self.filters(valid_filters)
criteria_filters = {}
tags = filters.get('tag', [])
if tags:
criteria_filters['tags'] = {'$all': filters.get('tag', [])}
criteria = Criteria.from_client_input({'filters': criteria_filters})
serialized_task_statuses = []
for task in TaskStatusManager.find_by_criteria(criteria):
task.update(serialization.dispatch.spawned_tasks(task))
serialized_task_statuses.append(task)
return self.ok(serialized_task_statuses)
def POST(self):
"""
Determine content applicability.
body {
consumer_criteria:<dict> or None,
repo_criteria:<dict> or None,
units: {<type_id1> : [{<unit1>}, {<unit2}, ..]
<type_id2> : [{<unit1>}, {<unit2}, ..]} or None
}
:return: A dict of applicability reports keyed by consumer ID.
Each consumer report is:
{ <unit_type_id1> : [<ApplicabilityReport>],
<unit_type_id1> : [<ApplicabilityReport>]},
}
:rtype: dict
"""
body = self.params()
consumer_criteria = body.get('consumer_criteria', None)
repo_criteria = body.get('repo_criteria', None)
units = body.get('units', None)
if consumer_criteria:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
if repo_criteria:
repo_criteria = Criteria.from_client_input(repo_criteria)
manager = managers.consumer_applicability_manager()
report = manager.units_applicable(consumer_criteria, repo_criteria, units)
for consumer_report in report.values():
for unit_type_id, report_list in consumer_report.items():
consumer_report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in report_list]
return self.ok(report)
def test_profiler_no_exception(self):
# Setup
self.populate()
profiler, cfg = plugins.get_profiler_by_type('rpm')
profiler.find_applicable_units = Mock(side_effect=KeyError)
# Test
user_specified_unit_criteria = {'rpm': {"filters": {"name": {"$in":['zsh','ksh']}}},
'mock-type': {"filters": {"name": {"$in":['abc','def']}}}
}
unit_criteria = {}
for type_id, criteria in user_specified_unit_criteria.items():
unit_criteria[type_id] = Criteria.from_client_input(criteria)
manager = factory.consumer_applicability_manager()
result = manager.find_applicable_units(self.CONSUMER_CRITERIA, self.REPO_CRITERIA, unit_criteria)
self.assertTrue(result == {})
def _get_consumer_criteria(self):
"""
Process the POST data, finding the criteria given by the user, and resolve it to Criteria
object.
:return: A Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
body = self.params()
try:
consumer_criteria = body.get('criteria')
except AttributeError:
raise InvalidValue('The input to this method must be a JSON object with a '
"'criteria' key.")
consumer_criteria = Criteria.from_client_input(consumer_criteria)
return consumer_criteria
def _get_query_results_from_post(self):
"""
Looks for a Criteria passed as a POST parameter on ket 'criteria', and
returns the results of a search based on that Criteria.
@return: list of documents from the DB that match the given criteria
for the collection associated with this controller
@rtype: list
"""
try:
criteria_param = self.params()['criteria']
except KeyError:
raise exceptions.MissingValue(['criteria'])
criteria = Criteria.from_client_input(criteria_param)
if criteria.fields and 'id' not in criteria.fields and u'id' not in criteria.fields:
criteria.fields.append('id')
return list(self.query_method(criteria))
def post(self, request, consumer_group_id):
"""
Unassociate a consumer from the group.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param consumer_group_id: id for the requested group
:type consumer_group_id: str
:return: Response containing consumers bound to the group
:rtype: django.http.HttpResponse
"""
params = request.body_as_json
criteria = Criteria.from_client_input(params.get('criteria', {}))
manager = factory.consumer_group_manager()
manager.unassociate(consumer_group_id, criteria)
query_manager = factory.consumer_group_query_manager()
group = query_manager.get_group(consumer_group_id)
return generate_json_response_with_pulp_encoder(group['consumer_ids'])
def post(self, request, repo_group_id):
"""
Unassociate repos that match criteria specified in the body to the specified repo group.
Call is idempotent.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param repo_group_id: matching repos are unassociated with this repo group
:type repo_group_id: str
:return: Response containing a serialized list of unassociated repository names
:rtype: django.http.HttpResponse
"""
criteria = Criteria.from_client_input(request.body_as_json.get('criteria', {}))
manager = managers_factory.repo_group_manager()
manager.unassociate(repo_group_id, criteria)
collection = RepoGroupModel.get_collection()
group = collection.find_one({'id': repo_group_id})
return generate_json_response(group['repo_ids'])
def POST(self):
"""
Creates an async task to regenerate content applicability data for given consumers.
body {consumer_criteria:<dict>}
"""
body = self.params()
consumer_criteria = body.get('consumer_criteria', None)
if consumer_criteria is None:
raise MissingValue('consumer_criteria')
try:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
except:
raise InvalidValue('consumer_criteria')
task_tags = [tags.action_tag('content_applicability_regeneration')]
async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
(consumer_criteria.as_dict(),), tags=task_tags)
raise OperationPostponed(async_result)
def POST(self):
"""
Creates an async task to regenerate content applicability data for given updated
repositories.
body {repo_criteria:<dict>}
"""
body = self.params()
repo_criteria = body.get('repo_criteria', None)
if repo_criteria is None:
raise exceptions.MissingValue('repo_criteria')
try:
repo_criteria = Criteria.from_client_input(repo_criteria)
except:
raise exceptions.InvalidValue('repo_criteria')
regeneration_tag = tags.action_tag('content_applicability_regeneration')
async_result = regenerate_applicability_for_repos.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
(repo_criteria.as_dict(),), tags=[regeneration_tag])
raise exceptions.OperationPostponed(async_result)
def _get_consumer_criteria(self, request):
"""
Process the POST data, finding the criteria given by the user, and resolve it to Criteria
object.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:raises InvalidValue: if some parameters were invalid
:return: A Criteria object
:rtype: pulp.server.db.model.criteria.Criteria
"""
body = request.body_as_json
consumer_criteria = body.get('criteria')
if consumer_criteria is None:
raise InvalidValue("The input to this method must be a JSON object with a "
"'criteria' key.")
consumer_criteria = Criteria.from_client_input(consumer_criteria)
return consumer_criteria
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async((repo_id, batch),
**{'group_id': task_group_id})
return task_group_id
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
# the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
# more details.
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}).batch_size(5)
for existing_applicability in existing_applicabilities:
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
projection=['id',
'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def regenerate_applicability_for_consumers(consumer_criteria):
"""
Regenerate and save applicability data for given updated consumers.
:param consumer_criteria: The consumer selection criteria
:type consumer_criteria: dict
"""
consumer_criteria = Criteria.from_dict(consumer_criteria)
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
consumer_profile_manager = managers.consumer_profile_manager()
# Process consumer_criteria and get all the consumer ids satisfied by the criteria
consumer_criteria.fields = ['id']
consumer_ids = [c['id'] for c in consumer_query_manager.find_by_criteria(consumer_criteria)]
# Following logic of checking existing applicability and getting required data
# to generate applicability is a bit more complicated than what it could be 'by design'.
# It is to optimize the number of db queries and improving applicability generation
# performance. Please consider the implications for applicability generation time
# when making any modifications to this code.
# Get all unit profiles associated with given consumers
unit_profile_criteria = Criteria(
filters={'consumer_id': {'$in': consumer_ids}},
fields=['consumer_id', 'profile_hash', 'content_type', 'id'])
all_unit_profiles = consumer_profile_manager.find_by_criteria(unit_profile_criteria)
# Create a consumer-profile map with consumer id as the key and list of tuples
# with profile details as the value
consumer_unit_profiles_map = {}
# Also create a map of profile_id keyed by profile_hash for profile lookup.
profile_hash_profile_id_map = {}
for unit_profile in all_unit_profiles:
profile_hash = unit_profile['profile_hash']
content_type = unit_profile['content_type']
consumer_id = unit_profile['consumer_id']
profile_id = unit_profile['id']
profile_tuple = (profile_hash, content_type)
# Add this tuple to the list of profile tuples for a consumer
consumer_unit_profiles_map.setdefault(consumer_id, []).append(profile_tuple)
# We need just one profile_id per profile_hash to be used in regenerate_applicability
# method to get the actual profile corresponding to given profile_hash.
if profile_hash not in profile_hash_profile_id_map:
profile_hash_profile_id_map[profile_hash] = profile_id
# Get all repos bound to given consumers
bind_criteria = Criteria(filters={'consumer_id': {'$in': consumer_ids}},
fields=['repo_id', 'consumer_id'])
all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)
# Create a repo-consumer map with repo_id as the key and consumer_id list as the value
repo_consumers_map = {}
for binding in all_repo_bindings:
repo_consumers_map.setdefault(binding['repo_id'], []).append(binding['consumer_id'])
# Create a set of (repo_id, (profile_hash, content_type))
repo_profile_hashes = set()
for repo_id, consumer_id_list in repo_consumers_map.items():
for consumer_id in consumer_id_list:
if consumer_id in consumer_unit_profiles_map:
for unit_profile_tuple in consumer_unit_profiles_map[consumer_id]:
repo_profile_hashes.add((repo_id, unit_profile_tuple))
# Iterate through each tuple in repo_profile_hashes set and regenerate applicability,
# if it doesn't exist. These are all guaranteed to be unique tuples because of the logic
# used to create maps and sets above, eliminating multiple unnecessary queries
# to check for existing applicability for same profiles.
manager = managers.applicability_regeneration_manager()
for repo_id, (profile_hash, content_type) in repo_profile_hashes:
# Check if applicability for given profile_hash and repo_id already exists
if ApplicabilityRegenerationManager._is_existing_applicability(repo_id, profile_hash):
continue
# If applicability does not exist, generate applicability data for given profile
# and repo id.
profile_id = profile_hash_profile_id_map[profile_hash]
manager.regenerate_applicability(profile_hash, content_type, profile_id, repo_id)
def POST(self):
"""
Determine content applicability.
body {
consumer_criteria:<dict> or None,
repo_criteria:<dict> or None,
unit_criteria: <dict of type_id : unit_criteria> or None,
override_config: <dict> or None
}
:return:
When report_style is 'by_consumer' -
A dict of applicability reports keyed by consumer ID.
Each consumer report is:
{ <unit_type_id1> : [<ApplicabilityReport>],
<unit_type_id1> : [<ApplicabilityReport>]},
}
When report_style is 'by_units' -
A dict of <unit_type_id1>: [<ApplicabilityReport>]
where applicability_report.summary contains a list of applicable consumer ids.
:rtype: dict
"""
body = self.params()
consumer_criteria = body.get('consumer_criteria', None)
repo_criteria = body.get('repo_criteria', None)
units = body.get('unit_criteria', None)
override_config = body.get('override_config', None)
if consumer_criteria:
consumer_criteria = Criteria.from_client_input(consumer_criteria)
if repo_criteria:
repo_criteria = Criteria.from_client_input(repo_criteria)
# If unit_criteria is not specified, consider all units of all types
if not units:
units = {}
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
units[unit_type_id] = {}
# Validate user defined criteria and convert them to Criteria objects
unit_criteria = {}
for type_id, criteria in units.items():
if criteria is None:
criteria = {}
unit_criteria[type_id] = Criteria.from_client_input(criteria)
manager = managers.consumer_applicability_manager()
report = manager.find_applicable_units(consumer_criteria, repo_criteria, unit_criteria, override_config)
for unit_type_id, applicability_reports in report.items():
if isinstance(applicability_reports, list):
report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in applicability_reports]
else:
for consumer_id, report_list in applicability_reports.items():
report[unit_type_id][consumer_id] = [serialization.consumer.applicability_report(r) for r in report_list]
return self.ok(report)