示例#1
0
 def test_find_by_criteria(self, mock_repo_qs):
     self.populate()
     manager = factory.consumer_bind_manager()
     manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID,
                  self.NOTIFY_AGENT, self.BINDING_CONFIG)
     # Test
     criteria = Criteria({'consumer_id': self.CONSUMER_ID})
     bindings = manager.find_by_criteria(criteria)
     bind = bindings[0]
     self.assertEqual(len(bindings), 1)
     self.assertEqual(bind['consumer_id'], self.CONSUMER_ID)
     self.assertEqual(bind['repo_id'], self.REPO_ID)
     self.assertEqual(bind['distributor_id'], self.DISTRIBUTOR_ID)
     self.assertEqual(bind['notify_agent'], self.NOTIFY_AGENT)
     self.assertEqual(bind['binding_config'], self.BINDING_CONFIG)
     # Test ($in)
     criteria = Criteria({'consumer_id': {'$in': [self.CONSUMER_ID]}})
     bindings = manager.find_by_criteria(criteria)
     bind = bindings[0]
     self.assertEqual(len(bindings), 1)
     self.assertEqual(bind['consumer_id'], self.CONSUMER_ID)
     self.assertEqual(bind['repo_id'], self.REPO_ID)
     self.assertEqual(bind['distributor_id'], self.DISTRIBUTOR_ID)
     self.assertEqual(bind['notify_agent'], self.NOTIFY_AGENT)
     self.assertEqual(bind['binding_config'], self.BINDING_CONFIG)
示例#2
0
文件: consumers.py 项目: alanoe/pulp
    def post(self, request, consumer_id):
        """
        Creates an async task to regenerate content applicability data for given consumer.

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest
        :param consumer_id: The consumer ID.
        :type consumer_id: str

        :raises MissingResource: if some parameters are missing
        :raises OperationPostponed: when an async operation is performed.
        """

        consumer_query_manager = factory.consumer_query_manager()
        if consumer_query_manager.find_by_id(consumer_id) is None:
            raise MissingResource(consumer_id=consumer_id)
        consumer_criteria = Criteria(filters={'consumer_id': consumer_id})

        task_tags = [tags.action_tag('consumer_content_applicability_regeneration')]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_CONSUMER_TYPE,
            consumer_id,
            (consumer_criteria.as_dict(),),
            tags=task_tags)
        raise OperationPostponed(async_result)
示例#3
0
    def post(self, request, consumer_id):
        """
        Creates an async task to regenerate content applicability data for given consumer.

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest
        :param consumer_id: The consumer ID.
        :type consumer_id: str

        :raises MissingResource: if some parameters are missing
        :raises OperationPostponed: when an async operation is performed.
        """

        consumer_query_manager = factory.consumer_query_manager()
        if consumer_query_manager.find_by_id(consumer_id) is None:
            raise MissingResource(consumer_id=consumer_id)
        consumer_criteria = Criteria(filters={'consumer_id': consumer_id})

        task_tags = [
            tags.action_tag('consumer_content_applicability_regeneration')
        ]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_CONSUMER_TYPE,
            consumer_id, (consumer_criteria.as_dict(), ),
            tags=task_tags)
        raise OperationPostponed(async_result)
示例#4
0
class RepoGroupAssociationTests(base.PulpWebserviceTests):
    def setUp(self):
        super(RepoGroupAssociationTests, self).setUp()
        self.manager = manager_factory.repo_group_manager()

    def clean(self):
        super(RepoGroupAssociationTests, self).clean()
        RepoGroup.get_collection().remove()

    @mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
    @mock.patch(
        'pulp.server.managers.repo.group.cud.RepoGroupManager.associate')
    def test_associate(self, mock_associate, mock_from_client):
        # the CallRequest stuff made mocking out a repo group very difficult,
        # so I punted on that.
        self.manager.create_repo_group('rg1')

        post_data = {'criteria': {'filters': {'id': {'$in': ['repo1']}}}}
        status, body = self.post('/v2/repo_groups/rg1/actions/associate/',
                                 post_data)
        self.assertEqual(status, 200)

        self.assertEqual(mock_associate.call_count, 1)
        call_args = mock_associate.call_args[0]
        self.assertEqual(call_args[0], 'rg1')
        # verify that it created and used a Criteria instance
        self.assertEqual(call_args[1], mock_from_client.return_value)
        self.assertEqual(mock_from_client.call_args[0][0],
                         {'filters': {
                             'id': {
                                 '$in': ['repo1']
                             }
                         }})

    @mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
    @mock.patch(
        'pulp.server.managers.repo.group.cud.RepoGroupManager.unassociate')
    def test_unassociate(self, mock_unassociate, mock_from_client):
        # the CallRequest stuff made mocking out a repo group very difficult,
        # so I punted on that.
        self.manager.create_repo_group('rg1')

        post_data = {'criteria': {'filters': {'id': {'$in': ['repo1']}}}}
        status, body = self.post('/v2/repo_groups/rg1/actions/unassociate/',
                                 post_data)
        self.assertEqual(status, 200)

        self.assertEqual(mock_unassociate.call_count, 1)
        call_args = mock_unassociate.call_args[0]
        self.assertEqual(call_args[0], 'rg1')
        # verify that it created and used a Criteria instance
        self.assertEqual(call_args[1], mock_from_client.return_value)
        self.assertEqual(mock_from_client.call_args[0][0],
                         {'filters': {
                             'id': {
                                 '$in': ['repo1']
                             }
                         }})
示例#5
0
    def POST(self):
        """
        Determine content applicability.
        body {
        consumer_criteria:<dict> or None, 
        repo_criteria:<dict> or None, 
        unit_criteria: <dict of type_id : unit_criteria> or None,
        override_config: <dict> or None
        }

        :return: 

        When report_style is 'by_consumer' -
        A dict of applicability reports keyed by consumer ID.
            Each consumer report is:
                { <unit_type_id1> : [<ApplicabilityReport>],
                  <unit_type_id1> : [<ApplicabilityReport>]},
                }

        When report_style is 'by_units' -
        A dict of <unit_type_id1>: [<ApplicabilityReport>]
        where applicability_report.summary contains a list of applicable consumer ids.

        :rtype: dict
        """
        body = self.params()

        consumer_criteria = body.get('consumer_criteria', None)
        repo_criteria = body.get('repo_criteria', None)
        units = body.get('unit_criteria', None)
        override_config = body.get('override_config', None)

        if consumer_criteria:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)

        if repo_criteria:
            repo_criteria = Criteria.from_client_input(repo_criteria)

        unit_criteria = {}
        if units:
            for type_id, criteria in units.items():
                # process if criteria is not empty
                if criteria:
                    unit_criteria[type_id] = Criteria.from_client_input(criteria)
                else:
                    unit_criteria[type_id] = criteria

        manager = managers.consumer_applicability_manager()
        report = manager.find_applicable_units(consumer_criteria, repo_criteria, unit_criteria, override_config)

        for unit_type_id, applicability_reports in report.items():
            if isinstance(applicability_reports, list):
                report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in applicability_reports]
            else:
                for consumer_id, report_list in applicability_reports.items():
                    report[unit_type_id][consumer_id] = [serialization.consumer.applicability_report(r) for r in report_list]

        return self.ok(report)
class ConsumerGroupAssociationTests(base.PulpWebserviceTests):
    def setUp(self):
        super(ConsumerGroupAssociationTests, self).setUp()
        self.manager = managers.consumer_group_manager()

    def clean(self):
        super(ConsumerGroupAssociationTests, self).clean()
        ConsumerGroup.get_collection().remove()

    @mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
    @mock.patch(
        'pulp.server.managers.consumer.group.cud.ConsumerGroupManager.associate'
    )
    def test_associate(self, mock_associate, mock_from_client):
        self.manager.create_consumer_group('cg1')

        post_data = {'criteria': {'filters': {'id': {'$in': ['consumer1']}}}}
        status, body = self.post('/v2/consumer_groups/cg1/actions/associate/',
                                 post_data)
        self.assertEqual(status, 200)

        self.assertEqual(mock_associate.call_count, 1)
        call_args = mock_associate.call_args[0]
        self.assertEqual(call_args[0], 'cg1')
        # verify that it created and used a Criteria instance
        self.assertEqual(call_args[1], mock_from_client.return_value)
        self.assertEqual(mock_from_client.call_args[0][0],
                         {'filters': {
                             'id': {
                                 '$in': ['consumer1']
                             }
                         }})

    @mock.patch.object(Criteria, 'from_client_input', return_value=Criteria())
    @mock.patch(
        'pulp.server.managers.consumer.group.cud.ConsumerGroupManager.unassociate'
    )
    def test_unassociate(self, mock_unassociate, mock_from_client):
        self.manager.create_consumer_group('cg1')

        post_data = {'criteria': {'filters': {'id': {'$in': ['consumer1']}}}}
        status, body = self.post(
            '/v2/consumer_groups/cg1/actions/unassociate/', post_data)
        self.assertEqual(status, 200)

        self.assertEqual(mock_unassociate.call_count, 1)
        call_args = mock_unassociate.call_args[0]
        self.assertEqual(call_args[0], 'cg1')
        # verify that it created and used a Criteria instance
        self.assertEqual(call_args[1], mock_from_client.return_value)
        self.assertEqual(mock_from_client.call_args[0][0],
                         {'filters': {
                             'id': {
                                 '$in': ['consumer1']
                             }
                         }})
示例#7
0
    def __populate_units(self, unit_ids_by_type, repo_ids):
        """
        Parse a dictionary of unit ids keyed by content type id and populate units for each type
        if the criteria is empty.

        :param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
        :type unit_ids_by_type: dict

        :return: if units are specified, return the corresponding units. If unit_ids_by_type dict
                 is empty, return unit ids corresponging to all units in given repo ids.
                 If unit ids list for a particular unit type is empty, return all unit ids
                 in given repo ids with that unit type.
        :rtype: dict
        """
        repo_unit_association_query_manager = managers.repo_unit_association_query_manager(
        )
        result_units = {}

        if unit_ids_by_type is not None:
            for unit_type_id, unit_ids in unit_ids_by_type.items():
                # Get unit type specific collection
                if not unit_ids:
                    # If unit_list is empty for a unit_type, consider all units of specific type
                    criteria = Criteria(filters={
                        "repo_id": {
                            "$in": repo_ids
                        },
                        "unit_type_id": unit_type_id
                    },
                                        fields=['unit_id'])
                    repo_units = repo_unit_association_query_manager.find_by_criteria(
                        criteria)
                    pulp_unit_ids = [u['unit_id'] for u in repo_units]
                    result_units.setdefault(unit_type_id, []).extend(
                        list(set(pulp_unit_ids)))
                else:
                    result_units.setdefault(unit_type_id, []).extend(unit_ids)
        else:
            # If units are not specified, consider all units in given repos.
            all_unit_type_ids = content_types_db.all_type_ids()
            for unit_type_id in all_unit_type_ids:
                criteria = Criteria(filters={
                    "repo_id": {
                        "$in": repo_ids
                    },
                    "unit_type_id": unit_type_id
                },
                                    fields=['unit_id'])
                repo_units = repo_unit_association_query_manager.find_by_criteria(
                    criteria)
                pulp_unit_ids = [u['unit_id'] for u in repo_units]
                result_units.setdefault(unit_type_id,
                                        []).extend(list(set(pulp_unit_ids)))

        return result_units
示例#8
0
    def check_workers(self):
        """
        Look for missing workers, and dispatch a cleanup task if one goes missing.

        To find a missing worker, filter the Workers model for entries older than
        utcnow() - WORKER_TIMEOUT_SECONDS. The heartbeat times are stored in native UTC, so this is
        a comparable datetime.

        For each missing worker found, dispatch a _delete_worker task requesting that the resource
        manager delete the Worker and cleanup any associated work.

        This method logs and the debug and error levels.
        """
        msg = _('Looking for workers missing for more than %s seconds'
                ) % self.WORKER_TIMEOUT_SECONDS
        _logger.debug(msg)
        oldest_heartbeat_time = datetime.utcnow() - timedelta(
            seconds=self.WORKER_TIMEOUT_SECONDS)
        worker_criteria = Criteria(
            filters={'last_heartbeat': {
                '$lt': oldest_heartbeat_time
            }},
            fields=('_id', 'last_heartbeat', 'num_reservations'))
        worker_list = list(resources.filter_workers(worker_criteria))
        for worker in worker_list:
            msg = _(
                "Workers '%s' has gone missing, removing from list of workers"
            ) % worker.name
            _logger.error(msg)
            _delete_worker.apply_async(args=(worker.name, ),
                                       queue=RESOURCE_MANAGER_QUEUE)
示例#9
0
    def post(self, request):
        """
        Dispatch a task to regenerate content applicability data for repositories that match
        the criteria passed in the body.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest

        :raises exceptions.MissingValue: if repo_critera is not a body parameter
        :raises exceptions.InvalidValue: if repo_critera (dict) has unsupported keys,
                                              the manager will raise an InvalidValue for the
                                              specific keys. Here, we create a parent exception
                                              for repo_criteria and include the specific keys as
                                              child exceptions.
        :raises exceptions.OperationPostponed: dispatch a task
        """
        class GroupCallReport(dict):
            def serialize(self):
                return self

        repo_criteria_body = request.body_as_json.get('repo_criteria', None)
        parallel = request.body_as_json.get('parallel', False)

        if repo_criteria_body is None:
            raise exceptions.MissingValue('repo_criteria')
        try:
            repo_criteria = Criteria.from_client_input(repo_criteria_body)
        except exceptions.InvalidValue, e:
            invalid_criteria = exceptions.InvalidValue('repo_criteria')
            invalid_criteria.add_child_exception(e)
            raise invalid_criteria
示例#10
0
    def post(self, request):
        """
        Dispatch a task to regenerate content applicability data for repositories that match
        the criteria passed in the body.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest

        :raises exceptions.MissingValue: if repo_critera is not a body parameter
        :raises exceptions.InvalidValue: if repo_critera (dict) has unsupported keys,
                                              the manager will raise an InvalidValue for the
                                              specific keys. Here, we create a parent exception
                                              for repo_criteria and include the specific keys as
                                              child exceptions.
        :raises exceptions.OperationPostponed: dispatch a task
        """
        class GroupCallReport(dict):
            def serialize(self):
                return self

        repo_criteria_body = request.body_as_json.get('repo_criteria', None)
        if repo_criteria_body is None:
            raise exceptions.MissingValue('repo_criteria')
        try:
            repo_criteria = Criteria.from_client_input(repo_criteria_body)
        except exceptions.InvalidValue, e:
            invalid_criteria = exceptions.InvalidValue('repo_criteria')
            invalid_criteria.add_child_exception(e)
            raise invalid_criteria
示例#11
0
def get_workers():
    """
    :returns:          list of workers with their heartbeats
    :rtype:            list
    """
    empty_criteria = Criteria()
    return resources.filter_workers(empty_criteria)
示例#12
0
    def regenerate_applicability_for_repos(repo_criteria):
        """
        Regenerate and save applicability data affected by given updated repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)
        repo_query_manager = managers.repo_query_manager()

        # Process repo criteria
        repo_criteria.fields = ['id']
        repo_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]

        for repo_id in repo_ids:
            # Find all existing applicabilities for given repo_id
            existing_applicabilities = RepoProfileApplicability.get_collection().find(
                {'repo_id': repo_id})
            for existing_applicability in existing_applicabilities:
                # Convert cursor to RepoProfileApplicability object
                existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
                profile_hash = existing_applicability['profile_hash']
                unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
                                                                     fields=['id', 'content_type'])
                if unit_profile is None:
                    # Unit profiles change whenever packages are installed or removed on consumers,
                    # and it is possible that existing_applicability references a UnitProfile
                    # that no longer exists. This is harmless, as Pulp has a monthly cleanup task
                    # that will identify these dangling references and remove them.
                    continue

                # Regenerate applicability data for given unit_profile and repo id
                ApplicabilityRegenerationManager.regenerate_applicability(
                    profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
                    existing_applicability)
示例#13
0
文件: content.py 项目: pombreda/pulp
    def get(self, request, type_id):
        """
        Return a response with a serialized list of the content units of the specified type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: the list of content units will be limited to this type
        :type  type_id: str

        :return: response with a serialized list of dicts, one for each unit of the type.
        :rtype: django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        all_units = cqm.find_by_criteria(type_id, Criteria())
        all_processed_units = []
        for unit in all_units:
            unit = serialization.content.content_unit_obj(unit)
            unit.update({
                '_href':
                '/'.join(
                    [request.get_full_path().rstrip('/'), unit['_id'], ''])
            })
            unit.update({
                'children':
                serialization.content.content_unit_child_link_objs(unit)
            })
            all_processed_units.append(unit)

        return generate_json_response_with_pulp_encoder(all_processed_units)
示例#14
0
    def _get_repo_consumer_map(consumer_ids=[], repo_ids=[]):
        """
        Create a repo-consumer map:
        {repo_id: [consumer_id1, consumer_id2,...]

        Data can be limited to specific consumers or to specific repos.

        :param consumer_ids: consumers which should be included
        :type  consumer_ids: list
        :param repo_ids: repositories which should be included
        :type  repo_ids: list

        :return: repo-consumer map described above
        :rtype: dict
        """
        bind_manager = managers.consumer_bind_manager()

        filters = {}
        if consumer_ids:
            filters = {'consumer_id': {'$in': consumer_ids}}
        elif repo_ids:
            filters = {'repo_id': {'$in': repo_ids}}

        bind_criteria = Criteria(filters=filters,
                                 fields=['repo_id', 'consumer_id'])
        all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)

        repo_consumers_map = {}
        for binding in all_repo_bindings:
            repo_consumers_map.setdefault(binding['repo_id'],
                                          []).append(binding['consumer_id'])

        return repo_consumers_map
示例#15
0
    def test_find_by_criteria_with_result(self):
        tags = ['test', 'tags']
        TaskStatus(task_id='1').save()
        TaskStatus(task_id='2', tags=tags).save()

        result = 'done'
        TaskStatus(task_id='3',
                   tags=tags,
                   state=constants.CALL_FINISHED_STATE,
                   result=result).save()

        filters = {'tags': tags, 'task_id': {'$in': ['1', '3']}}
        fields = ['task_id', 'tags', 'result']
        limit = 1
        sort = (('task_id', DESCENDING), )
        criteria = Criteria(filters=filters,
                            fields=fields,
                            limit=limit,
                            sort=sort)
        query_set = TaskStatus.objects.find_by_criteria(criteria)
        self.assertEqual(len(query_set), 1)
        self.assertEqual(query_set[0].task_id, '3')
        self.assertEqual(query_set[0].result, result)
        task_state_default = constants.CALL_WAITING_STATE
        self.assertEqual(query_set[0].state, task_state_default)
示例#16
0
文件: contents.py 项目: aweiteka/pulp
 def GET(self, type_id):
     """
     List all the available content units.
     """
     cqm = factory.content_query_manager()
     units = cqm.find_by_criteria(type_id, Criteria())
     return self.ok([self.process_unit(unit) for unit in units])
示例#17
0
文件: __init__.py 项目: werwty/pulp
    def translate_criteria(self, model, crit):
        """
        To preserve backwards compatability of our search API, we must translate the fields from
        the external representation to the internal representation. This is done most often with
        'id' since this is not an allowable key in the database in Mongoengine.

        This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
        be a dictionary containing the fields that have been renamed in the format:

        {'internal repr': 'external repr'}

        :param model: the class that defines this document's fields
        :type  model: sublcass of mongoengine.Document
        :param crit: criteria object to be translated from external to internal representation
        :type  crit: pulp.server.db.model.criteria.Criteria

        :return: translated Criteria object
        :rtype:  pulp.server.db.model.criteria.Criteria
        """
        # Circular import avoidance, since criteria imports models which imports serializers
        from pulp.server.db.model.criteria import Criteria
        crit_dict = crit.as_dict()
        if crit.filters:
            crit_dict['filters'] = self.translate_filters(model, crit.filters)
        if crit.sort:
            sort = [(self.translate_field(model, field), direc)
                    for field, direc in crit.sort]
            crit_dict['sort'] = sort
        if crit.fields:
            crit_dict['fields'] = [
                self.translate_field(model, field) for field in crit.fields
            ]
        return Criteria.from_dict(crit_dict)
示例#18
0
    def POST(self):
        """
        Creates an async task to regenerate content applicability data for given updated
        repositories.

        body {repo_criteria:<dict>}
        """
        body = self.params()
        repo_criteria = body.get('repo_criteria', None)
        if repo_criteria is None:
            raise exceptions.MissingValue('repo_criteria')
        try:
            repo_criteria = Criteria.from_client_input(repo_criteria)
        except:
            raise exceptions.InvalidValue('repo_criteria')

        manager = manager_factory.applicability_regeneration_manager()
        regeneration_tag = action_tag('applicability_regeneration')
        call_request = CallRequest(manager.regenerate_applicability_for_repos,
                                   [repo_criteria],
                                   tags = [regeneration_tag])
        # allow only one applicability regeneration task at a time
        call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE,
                                      dispatch_constants.RESOURCE_ANY_ID)
        return execution.execute_async(self, call_request)
示例#19
0
    def _get_total(self, id_list=None, ignore_filter=False):
        """
        Return the total number of units that are processed by this step.
        This is used generally for progress reporting.  The value returned should not change
        during the processing of the step.

        :param id_list: List of type ids to get the total count of
        :type id_list: list of str
        :param ignore_filter: Ignore the association filter and get all units of the given types
        :type ignore_filter: bool
        """
        if id_list is None:
            id_list = self.unit_type
        total = 0
        types_to_query = set(id_list).difference(self.skip_list)
        if not ignore_filter and self.association_filters:
            # We are copying using a filter so we have to get everything
            new_filter = copy.deepcopy(self.association_filters)
            new_filter['unit_type_id'] = {'$in': list(types_to_query)}
            criteria = Criteria(filters=new_filter)
            association_query_manager = manager_factory.repo_unit_association_query_manager(
            )
            units_cursor = association_query_manager.find_by_criteria(criteria)
            total = units_cursor.count()
        else:
            for type_id in types_to_query:
                total += self.parent.repo.content_unit_counts.get(type_id, 0)
        return total
示例#20
0
def handle_worker_heartbeat(event):
    """
    Celery event handler for 'worker-heartbeat' events.

    The event is first parsed and logged. If this event is from the resource manager, there is
    no further processing to be done. Then the existing Worker objects are searched
    for one to update. If an existing one is found, it is updated. Otherwise a new
    Worker entry is created. Logging at the info and debug level is also done.

    :param event: A celery event to handle.
    :type event: dict
    """
    event_info = _parse_and_log_event(event)

    # if this is the resource_manager do nothing
    if _is_resource_manager(event):
        return

    find_worker_criteria = Criteria(filters={'_id': event_info['worker_name']},
                                    fields=('_id', 'last_heartbeat',
                                            'num_reservations'))
    find_worker_list = list(resources.filter_workers(find_worker_criteria))

    if find_worker_list:
        Worker.get_collection().find_and_modify(
            query={'_id': event_info['worker_name']},
            update={'$set': {
                'last_heartbeat': event_info['timestamp']
            }})
    else:
        new_worker = Worker(event_info['worker_name'], event_info['timestamp'])
        msg = _("New worker '%(worker_name)s' discovered") % event_info
        _logger.info(msg)
        new_worker.save()
示例#21
0
文件: consumers.py 项目: ipanova/pulp
def expand_consumers(options, consumers):
    """
    Expand a list of users based on flags specified in the
    post body or query parameters.  The _href is always added by the
    serialization function used.
    Supported options:
      details - synonym for: (bindings=True,)
      bindings - include bindings
    @param options: The (expanding) options.
    @type options: dict
    @param consumers: A list of consumers
    @type consumers: list
    @return: A list of expanded consumers.
    @rtype: list
    """
    if options.get('details', False):
        options['bindings'] = True
    # add bindings
    if options.get('bindings', False):
        ids = [c['id'] for c in consumers]
        manager = managers.consumer_bind_manager()
        criteria = Criteria({'consumer_id': {'$in': ids}})
        bindings = manager.find_by_criteria(criteria)
        collated = {}
        for b in bindings:
            lst = collated.setdefault(b['consumer_id'], [])
            lst.append(b)
        for _consumer in consumers:
            _consumer['bindings'] = \
                [serialization.binding.serialize(b, False)
                    for b in collated.get(_consumer['id'], [])]
    return consumers
示例#22
0
文件: repo_groups.py 项目: omps/pulp
 def POST(self, repo_group_id):
     criteria = Criteria.from_client_input(self.params().get('criteria', {}))
     manager = managers_factory.repo_group_manager()
     manager.unassociate(repo_group_id, criteria)
     collection = RepoGroup.get_collection()
     group = collection.find_one({'id': repo_group_id})
     return self.ok(group['repo_ids'])
示例#23
0
文件: search.py 项目: stpierre/pulp
    def _get_query_results_from_get(self, ignore_fields=None):
        """
        Looks for query parameters that define a Criteria, and returns the
        results of a search based on that Criteria.

        @param ignore_fields:   Field names to ignore. All other fields will be
                                used in an attempt to generate a Criteria
                                instance, which will fail if unexpected field
                                names are present.
        @type  ignore_fields:   list

        @return:    list of documents from the DB that match the given criteria
                    for the collection associated with this controller
        @rtype:     list
        """
        input = web.input(field=[])
        if ignore_fields:
            for field in ignore_fields:
                input.pop(field, None)

        # rename this to 'fields' within the dict, and omit it if empty so we
        # default to getting all fields
        fields = input.pop('field')
        if fields:
            if 'id' not in fields and u'id' not in fields:
                fields.append('id')
            input['fields'] = fields

        criteria = Criteria.from_client_input(input)
        return list(self.query_method(criteria))
示例#24
0
    def queue_regenerate_applicability_for_repos(repo_criteria):
        """
        Queue a group of tasks to generate and save applicability data affected by given updated
        repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)

        # Process repo criteria
        repo_criteria.fields = ['id']
        repo_ids = [
            r.repo_id
            for r in model.Repository.objects.find_by_criteria(repo_criteria)
        ]

        task_group_id = uuid4()

        for repo_id in repo_ids:
            profile_hashes = RepoProfileApplicability.get_collection().find(
                {'repo_id': repo_id}, {'profile_hash': 1})
            for batch in paginate(profile_hashes, 10):
                batch_regenerate_applicability_task.apply_async(
                    (repo_id, batch), **{'group_id': task_group_id})
        return task_group_id
示例#25
0
    def post(self, request):
        """
        Creates an async task to regenerate content applicability data for given consumers.

        body {consumer_criteria:<dict>}

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest

        :raises MissingValue: if some parameters are missing
        :raises InvalidValue: if some parameters are invalid
        :raises OperationPostponed: when an async operation is performed.
        """

        body = request.body_as_json
        consumer_criteria = body.get('consumer_criteria', None)
        if consumer_criteria is None:
            raise MissingValue('consumer_criteria')
        try:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)
        except:
            raise InvalidValue('consumer_criteria')

        task_tags = [tags.action_tag('content_applicability_regeneration')]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE,
            tags.RESOURCE_ANY_ID, (consumer_criteria.as_dict(), ),
            tags=task_tags)
        raise OperationPostponed(async_result)
示例#26
0
 def test_profiler_no_exception(self):
     # Setup
     self.populate()
     profiler, cfg = plugins.get_profiler_by_type('rpm')
     profiler.find_applicable_units = Mock(side_effect=KeyError)
     # Test
     user_specified_unit_criteria = {
         'rpm': {
             "filters": {
                 "name": {
                     "$in": ['zsh', 'ksh']
                 }
             }
         },
         'mock-type': {
             "filters": {
                 "name": {
                     "$in": ['abc', 'def']
                 }
             }
         }
     }
     unit_criteria = {}
     for type_id, criteria in user_specified_unit_criteria.items():
         unit_criteria[type_id] = Criteria.from_client_input(criteria)
     manager = factory.consumer_applicability_manager()
     result = manager.find_applicable_units(self.CONSUMER_CRITERIA,
                                            self.REPO_CRITERIA,
                                            unit_criteria)
     self.assertTrue(result == {})
示例#27
0
 def test_no_exception_for_profiler_notfound(self):
     # Setup
     self.populate()
     # Test
     user_specified_unit_criteria = {
         'rpm': {
             "filters": {
                 "name": {
                     "$in": ['zsh']
                 }
             }
         },
         'xxx': {
             "filters": {
                 "name": {
                     "$in": ['abc']
                 }
             }
         }
     }
     unit_criteria = {}
     for type_id, criteria in user_specified_unit_criteria.items():
         unit_criteria[type_id] = Criteria.from_client_input(criteria)
     manager = factory.consumer_applicability_manager()
     result = manager.find_applicable_units(self.CONSUMER_CRITERIA,
                                            self.REPO_CRITERIA,
                                            unit_criteria)
     self.assertTrue(result == {})
示例#28
0
文件: __init__.py 项目: grizax/pulp
    def translate_criteria(self, model, crit):
        """
        To preserve backwards compatability of our search API, we must translate the fields from
        the external representation to the internal representation. This is done most often with
        'id' since this is not an allowable key in the database in Mongoengine.

        This method relies on a map defined in the subclass's Meta: `remapped_fields` which should
        be a dictionary containing the fields that have been renamed in the format:

        {'internal repr': 'external repr'}

        :param model: the class that defines this document's fields
        :type  model: sublcass of mongoengine.Document
        :param crit: criteria object to be translated from external to internal representation
        :type  crit: pulp.server.db.model.criteria.Criteria

        :return: translated Criteria object
        :rtype:  pulp.server.db.model.criteria.Criteria
        """
        # Circular import avoidance, since criteria imports models which imports serializers
        from pulp.server.db.model.criteria import Criteria
        crit_dict = crit.as_dict()
        if crit.filters:
            crit_dict['filters'] = self._translate_filters(model, crit.filters)
        if crit.sort:
            sort = [(self._translate(model, field), direc) for field, direc in crit.sort]
            crit_dict['sort'] = sort
        if crit.fields:
            crit_dict['fields'] = [self._translate(model, field) for field in crit.fields]
        return Criteria.from_dict(crit_dict)
示例#29
0
 def POST(self, consumer_group_id):
     criteria = Criteria.from_client_input(self.params().get('criteria', {}))
     manager = managers_factory.consumer_group_manager()
     manager.unassociate(consumer_group_id, criteria)
     query_manager = managers_factory.consumer_group_query_manager()
     group = query_manager.get_group(consumer_group_id)
     return self.ok(group['consumer_ids'])
示例#30
0
文件: consumers.py 项目: alanoe/pulp
    def post(self, request):
        """
        Creates an async task to regenerate content applicability data for given consumers.

        body {consumer_criteria:<dict>}

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest

        :raises MissingValue: if some parameters are missing
        :raises InvalidValue: if some parameters are invalid
        :raises OperationPostponed: when an async operation is performed.
        """

        body = request.body_as_json
        consumer_criteria = body.get('consumer_criteria', None)
        if consumer_criteria is None:
            raise MissingValue('consumer_criteria')
        try:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)
        except:
            raise InvalidValue('consumer_criteria')

        task_tags = [tags.action_tag('content_applicability_regeneration')]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
            (consumer_criteria.as_dict(),), tags=task_tags)
        raise OperationPostponed(async_result)
示例#31
0
    def test_filter(self):
        """
        Test a filter operation to make sure the results appear to be correct.
        """
        # Make three workers. We'll filter for two of them.
        now = datetime.utcnow()
        kw_1 = Worker('worker_1', now)
        kw_1.save()
        kw_2 = Worker('worker_2', now)
        kw_2.save()
        kw_3 = Worker('worker_3', now)
        kw_3.save()
        criteria = Criteria(filters={'_id': {
            '$gt': 'worker_1'
        }},
                            sort=[('_id', pymongo.ASCENDING)])

        workers = resources.filter_workers(criteria)

        # Let's assert that workers is a generator, and then let's cast it to a list so it's easier
        # to test that we got the correct instances back.
        self.assertEqual(type(workers), types.GeneratorType)
        workers = list(workers)
        self.assertEqual(all([isinstance(w, Worker) for w in workers]), True)
        self.assertEqual(workers[0].name, 'worker_2')
        self.assertEqual(workers[1].name, 'worker_3')
示例#32
0
    def regenerate_applicability_for_repos(repo_criteria):
        """
        Regenerate and save applicability data affected by given updated repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)
        repo_query_manager = managers.repo_query_manager()

        # Process repo criteria
        repo_criteria.fields = ["id"]
        repo_ids = [r["id"] for r in repo_query_manager.find_by_criteria(repo_criteria)]

        for repo_id in repo_ids:
            # Find all existing applicabilities for given repo_id
            existing_applicabilities = RepoProfileApplicability.get_collection().find({"repo_id": repo_id})
            for existing_applicability in existing_applicabilities:
                # Convert cursor to RepoProfileApplicability object
                existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
                profile_hash = existing_applicability["profile_hash"]
                unit_profile = UnitProfile.get_collection().find_one(
                    {"profile_hash": profile_hash}, fields=["id", "content_type"]
                )
                # Regenerate applicability data for given unit_profile and repo id
                ApplicabilityRegenerationManager.regenerate_applicability(
                    profile_hash, unit_profile["content_type"], unit_profile["id"], repo_id, existing_applicability
                )
示例#33
0
    def _add_repo_memberships(units, type_id):
        """
        For a list of units, find what repos each is a member of, and add a list
        of repo_ids to each unit.

        :param units:   list of unit documents
        :type  units:   list of dicts
        :param type_id: content type id
        :type  type_id: str
        :return:    same list of units that was passed in, only for convenience.
                    units are modified in-place
        """
        # quick return if there is nothing to do
        if not units:
            return units

        unit_ids = [unit['_id'] for unit in units]
        criteria = Criteria(
            filters={'unit_id': {'$in': unit_ids}, 'unit_type_id': type_id},
            fields=('repo_id', 'unit_id')
        )
        associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria)
        unit_ids = None
        criteria = None
        association_map = {}
        for association in associations:
            association_map.setdefault(association['unit_id'], set()).add(
                association['repo_id'])

        for unit in units:
            unit['repository_memberships'] = list(association_map.get(unit['_id'], []))
        return units
示例#34
0
 def POST(self, consumer_group_id):
     criteria = Criteria.from_client_input(self.params().get(
         'criteria', {}))
     manager = managers_factory.consumer_group_manager()
     manager.unassociate(consumer_group_id, criteria)
     query_manager = managers_factory.consumer_group_query_manager()
     group = query_manager.get_group(consumer_group_id)
     return self.ok(group['consumer_ids'])
示例#35
0
def _delete_worker(name, normal_shutdown=False):
    """
    Delete the Worker with _id name from the database, cancel any associated tasks and reservations

    If the worker shutdown normally, no message is logged, otherwise an error level message is
    logged. Default is to assume the worker did not shut down normally.

    Any resource reservations associated with this worker are cleaned up by this function.

    Any tasks associated with this worker are explicitly canceled.

    :param name:            The name of the worker you wish to delete. In the database, the _id
                            field is the name.
    :type  name:            basestring
    :param normal_shutdown: True if the worker shutdown normally, False otherwise.  Defaults to
                            False.
    :type normal_shutdown:  bool
    """
    if normal_shutdown is False:
        msg = _(
            'The worker named %(name)s is missing. Canceling the tasks in its queue.'
        )
        msg = msg % {'name': name}
        logger.error(msg)

    # Delete the worker document
    worker_list = list(
        resources.filter_workers(Criteria(filters={'_id': name})))
    if len(worker_list) > 0:
        worker_document = worker_list[0]
        worker_document.delete()

    # Delete all reserved_resource documents for the worker
    ReservedResource.get_collection().remove({'worker_name': name})

    # Cancel all of the tasks that were assigned to this worker's queue
    worker = Worker.from_bson({'_id': name})
    for task in TaskStatusManager.find_by_criteria(
            Criteria(
                filters={
                    'worker_name': worker.name,
                    'state': {
                        '$in': constants.CALL_INCOMPLETE_STATES
                    }
                })):
        cancel(task['task_id'])
 def populate(self):
     manager = managers.consumer_manager()
     for consumer_id in CONSUMER_IDS:
         manager.register(consumer_id)
     manager = managers.consumer_group_manager()
     manager.create_consumer_group(GROUP_ID)
     for consumer_id in CONSUMER_IDS:
         criteria = Criteria(filters={'id': consumer_id}, fields=['id'])
         manager.associate(GROUP_ID, criteria)
示例#37
0
def get_enabled():
    """
    Get schedules that are enabled, that is, their "enabled" attribute is True

    :return:    pymongo cursor of ScheduledCall database objects
    :rtype:     pymongo.cursor.Cursor
    """
    criteria = Criteria(filters={'enabled': True})
    return ScheduledCall.get_collection().query(criteria)
示例#38
0
文件: tasks.py 项目: aweiteka/pulp
def _delete_worker(name, normal_shutdown=False):
    """
    Delete the Worker with _id name from the database. This Task can only safely be
    performed by the resource manager at this time, so be sure to queue it in the
    RESOURCE_MANAGER_QUEUE.

    If the worker shutdown normally, no message is logged, otherwise an error level message is
    logged. Default is to assume the work did not shut down normally.

    :param name:            The name of the worker you wish to delete. In the database, the _id
                            field is the name.
    :type  name:            basestring
    :param normal_shutdown: True if the worker shutdown normally, False otherwise.  Defaults to
                            False.
    :type normal_shutdown:  bool
    """
    worker_list = list(
        resources.filter_workers(Criteria(filters={'_id': name})))
    if len(worker_list) == 0:
        # Potentially _delete_worker() may be called with the database not containing any entries.
        # https://bugzilla.redhat.com/show_bug.cgi?id=1091922
        return
    worker = worker_list[0]

    if normal_shutdown is False:
        msg = _(
            'The worker named %(name)s is missing. Canceling the tasks in its queue.'
        )
        msg = msg % {'name': worker.name}
        logger.error(msg)

    # Cancel all of the tasks that were assigned to this worker's queue
    for task in TaskStatusManager.find_by_criteria(
            Criteria(
                filters={
                    'queue': worker.queue_name,
                    'state': {
                        '$in': constants.CALL_INCOMPLETE_STATES
                    }
                })):
        cancel(task['task_id'])

    # Finally, delete the worker
    worker.delete()
示例#39
0
文件: consumers.py 项目: ipanova/pulp
    def POST(self, consumer_id):
        """
        Creates an async task to regenerate content applicability data for given consumer.

        :param consumer_id: The consumer ID.
        :type consumer_id: basestring
        """
        consumer_query_manager = managers.consumer_query_manager()
        if consumer_query_manager.find_by_id(consumer_id) is None:
            raise MissingResource(consumer_id=consumer_id)
        consumer_criteria = Criteria(filters={'consumer_id': consumer_id})

        task_tags = [tags.action_tag('consumer_content_applicability_regeneration')]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_CONSUMER_TYPE,
            consumer_id,
            (consumer_criteria.as_dict(),),
            tags=task_tags)
        raise OperationPostponed(async_result)
示例#40
0
 def POST(self, consumer_group_id):
     criteria = Criteria.from_client_input(self.params().get('criteria', {}))
     manager = managers_factory.consumer_group_manager()
     tags = [resource_tag(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id),
             action_tag('consumer_group_unassociate')]
     call_request = CallRequest(manager.unassociate,
                                [consumer_group_id, criteria],
                                tags=tags)
     call_request.updates_resource(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id)
     matched = execution.execute(call_request)
     return self.ok(matched)
示例#41
0
 def test_no_exception_for_profiler_notfound(self):
     # Setup
     self.populate()
     # Test
     user_specified_unit_criteria = {'rpm': {"filters": {"name": {"$in":['zsh']}}},
                      'xxx': {"filters": {"name": {"$in":['abc']}}}
                     }
     unit_criteria = {}
     for type_id, criteria in user_specified_unit_criteria.items():
         unit_criteria[type_id] = Criteria.from_client_input(criteria)
     manager = factory.consumer_applicability_manager()
     result = manager.find_applicable_units(self.CONSUMER_CRITERIA, self.REPO_CRITERIA, unit_criteria)
     self.assertTrue(result == {})
示例#42
0
 def POST(self, consumer_group_id):
     criteria = Criteria.from_client_input(self.params().get('criteria', {}))
     manager = managers_factory.consumer_group_manager()
     tags = [resource_tag(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id),
             action_tag('consumer_group_associate')]
     call_request = CallRequest(manager.associate,
                                [consumer_group_id, criteria],
                                tags=tags)
     call_request.updates_resource(dispatch_constants.RESOURCE_CONSUMER_GROUP_TYPE, consumer_group_id)
     execution.execute(call_request)
     collection = ConsumerGroup.get_collection()
     group = collection.find_one({'id': consumer_group_id})
     return self.ok(group['consumer_ids'])
示例#43
0
 def POST(self, repo_group_id):
     criteria = Criteria.from_client_input(self.params().get('criteria', {}))
     manager = managers_factory.repo_group_manager()
     tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_GROUP_TYPE, repo_group_id),
             action_tag('repo_group_unassociate')]
     call_request = CallRequest(manager.unassociate,
                                [repo_group_id, criteria],
                                tags=tags)
     call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_GROUP_TYPE, repo_group_id)
     execution.execute(call_request)
     collection = RepoGroup.get_collection()
     group = collection.find_one({'id': repo_group_id})
     return self.ok(group['repo_ids'])
示例#44
0
 def GET(self):
     valid_filters = ['tag']
     filters = self.filters(valid_filters)
     criteria_filters = {}
     tags = filters.get('tag', [])
     if tags:
         criteria_filters['tags'] = {'$all':  filters.get('tag', [])}
     criteria = Criteria.from_client_input({'filters': criteria_filters})
     serialized_task_statuses = []
     for task in TaskStatusManager.find_by_criteria(criteria):
         task.update(serialization.dispatch.spawned_tasks(task))
         serialized_task_statuses.append(task)
     return self.ok(serialized_task_statuses)
示例#45
0
文件: consumers.py 项目: graco/pulp
    def POST(self):
        """
        Determine content applicability.
        body {
        consumer_criteria:<dict> or None, 
        repo_criteria:<dict> or None, 
        units: {<type_id1> : [{<unit1>}, {<unit2}, ..]
                <type_id2> : [{<unit1>}, {<unit2}, ..]} or None
        }

        :return: A dict of applicability reports keyed by consumer ID.
            Each consumer report is:
                { <unit_type_id1> : [<ApplicabilityReport>],
                  <unit_type_id1> : [<ApplicabilityReport>]},
                }
                
        :rtype: dict
        """
        body = self.params()

        consumer_criteria = body.get('consumer_criteria', None)
        repo_criteria = body.get('repo_criteria', None)
        units = body.get('units', None)

        if consumer_criteria:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)

        if repo_criteria:
            repo_criteria = Criteria.from_client_input(repo_criteria)

        manager = managers.consumer_applicability_manager()
        report = manager.units_applicable(consumer_criteria, repo_criteria, units)

        for consumer_report in report.values():
            for unit_type_id, report_list in consumer_report.items():
                consumer_report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in report_list]
        return self.ok(report)
示例#46
0
 def test_profiler_no_exception(self):
     # Setup
     self.populate()
     profiler, cfg = plugins.get_profiler_by_type('rpm')
     profiler.find_applicable_units = Mock(side_effect=KeyError)
     # Test
     user_specified_unit_criteria = {'rpm': {"filters": {"name": {"$in":['zsh','ksh']}}},
                                     'mock-type': {"filters": {"name": {"$in":['abc','def']}}}
                                     }
     unit_criteria = {}
     for type_id, criteria in user_specified_unit_criteria.items():
         unit_criteria[type_id] = Criteria.from_client_input(criteria)
     manager = factory.consumer_applicability_manager()
     result = manager.find_applicable_units(self.CONSUMER_CRITERIA, self.REPO_CRITERIA, unit_criteria)
     self.assertTrue(result == {})
示例#47
0
    def _get_consumer_criteria(self):
        """
        Process the POST data, finding the criteria given by the user, and resolve it to Criteria
        object.

        :return: A Criteria object
        :rtype:  pulp.server.db.model.criteria.Criteria
        """
        body = self.params()

        try:
            consumer_criteria = body.get('criteria')
        except AttributeError:
            raise InvalidValue('The input to this method must be a JSON object with a '
                               "'criteria' key.")
        consumer_criteria = Criteria.from_client_input(consumer_criteria)
        return consumer_criteria
示例#48
0
文件: search.py 项目: stpierre/pulp
    def _get_query_results_from_post(self):
        """
        Looks for a Criteria passed as a POST parameter on ket 'criteria', and
        returns the results of a search based on that Criteria.

        @return:    list of documents from the DB that match the given criteria
                    for the collection associated with this controller
        @rtype:     list
        """
        try:
            criteria_param = self.params()['criteria']
        except KeyError:
            raise exceptions.MissingValue(['criteria'])
        criteria = Criteria.from_client_input(criteria_param)
        if criteria.fields and 'id' not in criteria.fields and u'id' not in criteria.fields:
            criteria.fields.append('id')
        return list(self.query_method(criteria))
示例#49
0
    def post(self, request, consumer_group_id):
        """
        Unassociate a consumer from the group.

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest
        :param consumer_group_id: id for the requested group
        :type consumer_group_id: str
        :return: Response containing consumers bound to the group
        :rtype: django.http.HttpResponse
        """
        params = request.body_as_json
        criteria = Criteria.from_client_input(params.get('criteria', {}))
        manager = factory.consumer_group_manager()
        manager.unassociate(consumer_group_id, criteria)
        query_manager = factory.consumer_group_query_manager()
        group = query_manager.get_group(consumer_group_id)
        return generate_json_response_with_pulp_encoder(group['consumer_ids'])
示例#50
0
    def post(self, request, repo_group_id):
        """
        Unassociate repos that match criteria specified in the body to the specified repo group.
        Call is idempotent.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param repo_group_id: matching repos are unassociated with this repo group
        :type  repo_group_id: str

        :return: Response containing a serialized list of unassociated repository names
        :rtype: django.http.HttpResponse
        """
        criteria = Criteria.from_client_input(request.body_as_json.get('criteria', {}))
        manager = managers_factory.repo_group_manager()
        manager.unassociate(repo_group_id, criteria)
        collection = RepoGroupModel.get_collection()
        group = collection.find_one({'id': repo_group_id})
        return generate_json_response(group['repo_ids'])
示例#51
0
    def POST(self):
        """
        Creates an async task to regenerate content applicability data for given consumers.

        body {consumer_criteria:<dict>}
        """
        body = self.params()
        consumer_criteria = body.get('consumer_criteria', None)
        if consumer_criteria is None:
            raise MissingValue('consumer_criteria')
        try:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)
        except:
            raise InvalidValue('consumer_criteria')

        task_tags = [tags.action_tag('content_applicability_regeneration')]
        async_result = regenerate_applicability_for_consumers.apply_async_with_reservation(
            tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
            (consumer_criteria.as_dict(),), tags=task_tags)
        raise OperationPostponed(async_result)
示例#52
0
文件: repositories.py 项目: beav/pulp
    def POST(self):
        """
        Creates an async task to regenerate content applicability data for given updated
        repositories.

        body {repo_criteria:<dict>}
        """
        body = self.params()
        repo_criteria = body.get('repo_criteria', None)
        if repo_criteria is None:
            raise exceptions.MissingValue('repo_criteria')
        try:
            repo_criteria = Criteria.from_client_input(repo_criteria)
        except:
            raise exceptions.InvalidValue('repo_criteria')

        regeneration_tag = tags.action_tag('content_applicability_regeneration')
        async_result = regenerate_applicability_for_repos.apply_async_with_reservation(
            tags.RESOURCE_REPOSITORY_PROFILE_APPLICABILITY_TYPE, tags.RESOURCE_ANY_ID,
            (repo_criteria.as_dict(),), tags=[regeneration_tag])
        raise exceptions.OperationPostponed(async_result)
示例#53
0
文件: consumers.py 项目: alanoe/pulp
    def _get_consumer_criteria(self, request):
        """
        Process the POST data, finding the criteria given by the user, and resolve it to Criteria
        object.

        :param request: WSGI request object
        :type request: django.core.handlers.wsgi.WSGIRequest

        :raises InvalidValue: if some parameters were invalid

        :return: A Criteria object
        :rtype:  pulp.server.db.model.criteria.Criteria
        """

        body = request.body_as_json

        consumer_criteria = body.get('criteria')
        if consumer_criteria is None:
            raise InvalidValue("The input to this method must be a JSON object with a "
                               "'criteria' key.")
        consumer_criteria = Criteria.from_client_input(consumer_criteria)
        return consumer_criteria
示例#54
0
    def queue_regenerate_applicability_for_repos(repo_criteria):
        """
        Queue a group of tasks to generate and save applicability data affected by given updated
        repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)

        # Process repo criteria
        repo_criteria.fields = ['id']
        repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]

        task_group_id = uuid4()

        for repo_id in repo_ids:
            profile_hashes = RepoProfileApplicability.get_collection().find(
                {'repo_id': repo_id}, {'profile_hash': 1})
            for batch in paginate(profile_hashes, 10):
                batch_regenerate_applicability_task.apply_async((repo_id, batch),
                                                                **{'group_id': task_group_id})
        return task_group_id
示例#55
0
    def regenerate_applicability_for_repos(repo_criteria):
        """
        Regenerate and save applicability data affected by given updated repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)

        # Process repo criteria
        repo_criteria.fields = ['id']
        repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]

        for repo_id in repo_ids:
            # Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
            # the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
            # more details.
            existing_applicabilities = RepoProfileApplicability.get_collection().find(
                {'repo_id': repo_id}).batch_size(5)
            for existing_applicability in existing_applicabilities:
                existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
                profile_hash = existing_applicability['profile_hash']
                unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
                                                                     projection=['id',
                                                                                 'content_type'])
                if unit_profile is None:
                    # Unit profiles change whenever packages are installed or removed on consumers,
                    # and it is possible that existing_applicability references a UnitProfile
                    # that no longer exists. This is harmless, as Pulp has a monthly cleanup task
                    # that will identify these dangling references and remove them.
                    continue

                # Regenerate applicability data for given unit_profile and repo id
                ApplicabilityRegenerationManager.regenerate_applicability(
                    profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
                    existing_applicability)
示例#56
0
    def regenerate_applicability_for_consumers(consumer_criteria):
        """
        Regenerate and save applicability data for given updated consumers.

        :param consumer_criteria: The consumer selection criteria
        :type consumer_criteria: dict
        """
        consumer_criteria = Criteria.from_dict(consumer_criteria)
        consumer_query_manager = managers.consumer_query_manager()
        bind_manager = managers.consumer_bind_manager()
        consumer_profile_manager = managers.consumer_profile_manager()

        # Process consumer_criteria and get all the consumer ids satisfied by the criteria
        consumer_criteria.fields = ['id']
        consumer_ids = [c['id'] for c in consumer_query_manager.find_by_criteria(consumer_criteria)]

        # Following logic of checking existing applicability and getting required data
        # to generate applicability is a bit more complicated than what it could be 'by design'.
        # It is to optimize the number of db queries and improving applicability generation
        # performance. Please consider the implications for applicability generation time
        # when making any modifications to this code.

        # Get all unit profiles associated with given consumers
        unit_profile_criteria = Criteria(
            filters={'consumer_id': {'$in': consumer_ids}},
            fields=['consumer_id', 'profile_hash', 'content_type', 'id'])
        all_unit_profiles = consumer_profile_manager.find_by_criteria(unit_profile_criteria)

        # Create a consumer-profile map with consumer id as the key and list of tuples
        # with profile details as the value
        consumer_unit_profiles_map = {}
        # Also create a map of profile_id keyed by profile_hash for profile lookup.
        profile_hash_profile_id_map = {}
        for unit_profile in all_unit_profiles:
            profile_hash = unit_profile['profile_hash']
            content_type = unit_profile['content_type']
            consumer_id = unit_profile['consumer_id']
            profile_id = unit_profile['id']

            profile_tuple = (profile_hash, content_type)
            # Add this tuple to the list of profile tuples for a consumer
            consumer_unit_profiles_map.setdefault(consumer_id, []).append(profile_tuple)

            # We need just one profile_id per profile_hash to be used in regenerate_applicability
            # method to get the actual profile corresponding to given profile_hash.
            if profile_hash not in profile_hash_profile_id_map:
                profile_hash_profile_id_map[profile_hash] = profile_id

        # Get all repos bound to given consumers
        bind_criteria = Criteria(filters={'consumer_id': {'$in': consumer_ids}},
                                 fields=['repo_id', 'consumer_id'])
        all_repo_bindings = bind_manager.find_by_criteria(bind_criteria)

        # Create a repo-consumer map with repo_id as the key and consumer_id list as the value
        repo_consumers_map = {}
        for binding in all_repo_bindings:
            repo_consumers_map.setdefault(binding['repo_id'], []).append(binding['consumer_id'])

        # Create a set of (repo_id, (profile_hash, content_type))
        repo_profile_hashes = set()
        for repo_id, consumer_id_list in repo_consumers_map.items():
            for consumer_id in consumer_id_list:
                if consumer_id in consumer_unit_profiles_map:
                    for unit_profile_tuple in consumer_unit_profiles_map[consumer_id]:
                        repo_profile_hashes.add((repo_id, unit_profile_tuple))

        # Iterate through each tuple in repo_profile_hashes set and regenerate applicability,
        # if it doesn't exist. These are all guaranteed to be unique tuples because of the logic
        # used to create maps and sets above, eliminating multiple unnecessary queries
        # to check for existing applicability for same profiles.
        manager = managers.applicability_regeneration_manager()
        for repo_id, (profile_hash, content_type) in repo_profile_hashes:
            # Check if applicability for given profile_hash and repo_id already exists
            if ApplicabilityRegenerationManager._is_existing_applicability(repo_id, profile_hash):
                continue
            # If applicability does not exist, generate applicability data for given profile
            # and repo id.
            profile_id = profile_hash_profile_id_map[profile_hash]
            manager.regenerate_applicability(profile_hash, content_type, profile_id, repo_id)
示例#57
0
    def POST(self):
        """
        Determine content applicability.
        body {
        consumer_criteria:<dict> or None, 
        repo_criteria:<dict> or None, 
        unit_criteria: <dict of type_id : unit_criteria> or None,
        override_config: <dict> or None
        }

        :return: 

        When report_style is 'by_consumer' -
        A dict of applicability reports keyed by consumer ID.
            Each consumer report is:
                { <unit_type_id1> : [<ApplicabilityReport>],
                  <unit_type_id1> : [<ApplicabilityReport>]},
                }

        When report_style is 'by_units' -
        A dict of <unit_type_id1>: [<ApplicabilityReport>]
        where applicability_report.summary contains a list of applicable consumer ids.

        :rtype: dict
        """
        body = self.params()

        consumer_criteria = body.get('consumer_criteria', None)
        repo_criteria = body.get('repo_criteria', None)
        units = body.get('unit_criteria', None)
        override_config = body.get('override_config', None)

        if consumer_criteria:
            consumer_criteria = Criteria.from_client_input(consumer_criteria)

        if repo_criteria:
            repo_criteria = Criteria.from_client_input(repo_criteria)

        # If unit_criteria is not specified, consider all units of all types
        if not units:
            units = {}
            all_unit_type_ids = content_types_db.all_type_ids()
            for unit_type_id in all_unit_type_ids:
                units[unit_type_id] = {}
        # Validate user defined criteria and convert them to Criteria objects
        unit_criteria = {}
        for type_id, criteria in units.items():
            if criteria is None:
                criteria = {}
            unit_criteria[type_id] = Criteria.from_client_input(criteria)

        manager = managers.consumer_applicability_manager()
        report = manager.find_applicable_units(consumer_criteria, repo_criteria, unit_criteria, override_config)

        for unit_type_id, applicability_reports in report.items():
            if isinstance(applicability_reports, list):
                report[unit_type_id] = [serialization.consumer.applicability_report(r) for r in applicability_reports]
            else:
                for consumer_id, report_list in applicability_reports.items():
                    report[unit_type_id][consumer_id] = [serialization.consumer.applicability_report(r) for r in report_list]

        return self.ok(report)