def validate_updated_schedule_options(options): """ Validate updated schedule options. :param options: updated options for a scheduled call :type options: dict :raises: pulp.server.exceptions.UnsupportedValue if unsupported schedule options are passed in :raises: pulp.server.exceptions.InvalidValue if any of the options are invalid """ unknown_options = _find_unknown_options(options, ScheduledCall.USER_UPDATE_FIELDS) if unknown_options: raise exceptions.UnsupportedValue(unknown_options) invalid_options = [] if 'iso_schedule' in options and not _is_valid_schedule( options['iso_schedule']): invalid_options.append('iso_schedule') if 'failure_threshold' in options and not _is_valid_failure_threshold( options['failure_threshold']): invalid_options.append('failure_threshold') if 'remaining_runs' in options and not _is_valid_remaining_runs( options['remaining_runs']): invalid_options.append('remaining_runs') if 'enabled' in options and not _is_valid_enabled_flag(options['enabled']): invalid_options.append('enabled') if not invalid_options: return raise exceptions.InvalidValue(invalid_options)
def validate_initial_schedule_options(schedule, failure_threshold, enabled): """ Validate the initial schedule and schedule options. :param options: options for the schedule :type options: dict :raises: pulp.server.exceptions.UnsupportedValue if unsupported schedule options are passed in :raises: pulp.server.exceptions.InvalidValue if any of the options are invalid """ invalid_options = [] if not _is_valid_schedule(schedule): invalid_options.append('schedule') if not _is_valid_failure_threshold(failure_threshold): invalid_options.append('failure_threshold') if not _is_valid_enabled_flag(enabled): invalid_options.append('enabled') if not invalid_options: return raise exceptions.InvalidValue(invalid_options)
def validate_keys(options, valid_keys, all_required=False): """ Validate the keys of a dictionary using the list of valid keys. :param options: dictionary of options to validate :type options: dict :param valid_keys: list of keys that are valid :type valid_keys: list or tuple :param all_required: flag whether all the keys in valid_keys must be present :type all_required: bool """ invalid_keys = [] for key in options: if key not in valid_keys: invalid_keys.append(key) if invalid_keys: raise exceptions.InvalidValue(invalid_keys) if not all_required: return missing_keys = [] for key in valid_keys: if key not in options: missing_keys.append(key) if missing_keys: raise exceptions.MissingValue(missing_keys)
def POST(self, role_id): # Params (validation will occur in the manager) params = self.params() login = params.get('login', None) if login is None: raise exceptions.InvalidValue(login) role_manager = managers.role_manager() resources = { dispatch_constants.RESOURCE_USER_TYPE: { login: dispatch_constants.RESOURCE_UPDATE_OPERATION } } tags = [ resource_tag(dispatch_constants.RESOURCE_ROLE_TYPE, role_id), action_tag('add_user_to_role') ] call_request = CallRequest(role_manager.add_user_to_role, [role_id, login], resources=resources, tags=tags) return self.ok(execution.execute_sync(call_request))
def update_importer_config(repo_id, importer_config): """ Attempts to update the saved configuration for the given repo's importer. The importer will be asked if the new configuration is valid. If not, this method will raise an error and the existing configuration will remain unchanged. :param repo_id: identifies the repo :type repo_id: str :param importer_config: new configuration values to use for this repo :type importer_config: dict """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) # MongoDB does not allow to change keys of the DictField and values of the same DictField # simultaneously thus we need to save repo_importer twice. for k, v in importer_config.iteritems(): if v is None: repo_importer.config.pop(k, None) # Config can be invalid if some options got removed but their complementary options did not, # e.g. basic_auth_username and basic_auth_password should be either both present or both absent. validate_importer_config(repo_obj, repo_importer.importer_type_id, repo_importer.config) repo_importer.save() for k, v in importer_config.iteritems(): if v is not None: repo_importer.config[k] = v validate_importer_config(repo_obj, repo_importer.importer_type_id, repo_importer.config) try: repo_importer.save() except ValidationError, e: raise exceptions.InvalidValue(e.to_dict().keys())
def create_user(login, password=None, name=None, roles=None): """ Creates a new Pulp user and adds it to specified to roles. :param login: login name / unique identifier for the user :type login: str :param password: password for login credentials :type password: str :param name: user's full name :type name: str :param roles: list of roles user will belong to :type roles: list :raise DuplicateResource: if there is already a user with the requested login :raise InvalidValue: if any of the fields are unacceptable """ user = model.User(login=login, name=name, roles=roles) user.set_password(password) try: user.save() except NotUniqueError: raise pulp_exceptions.DuplicateResource(login) except ValidationError, e: raise pulp_exceptions.InvalidValue(e.to_dict().keys())
def get(self, request, *args, **kwargs): """ Search for objects using an HTTP GET request. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :return: HttpReponse containing a list of objects that were matched by the request :rtype: django.http.HttpResponse :raises InvalidValue: if filters is passed but is not valid JSON """ query, options = self._parse_args(request.GET) filters = request.GET.get('filters') if filters: try: query['filters'] = json.loads(filters) except ValueError: raise exceptions.InvalidValue('filters') fields = query.pop('field', '') if fields: query['fields'] = fields return self._generate_response(query, options, *args, **kwargs)
def associate_from_repo(source_repo_id, dest_repo_id, criteria=None, import_config_override=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. :param source_repo_id: identifies the source repository :type source_repo_id: str :param dest_repo_id: identifies the destination repository :type dest_repo_id: str :param criteria: optional; if specified, will filter the units retrieved from the source repository :type criteria: UnitAssociationCriteria :param import_config_override: optional config containing values to use for this import only :type import_config_override: dict :return: dict with key 'units_successful' whose value is a list of unit keys that were copied. units that were associated by this operation :rtype: dict :raise MissingResource: if either of the specified repositories don't exist """ importer_manager = manager_factory.repo_importer_manager() source_repo = model.Repository.objects.get_repo_or_missing_resource( source_repo_id) dest_repo = model.Repository.objects.get_repo_or_missing_resource( dest_repo_id) # This will raise MissingResource if there isn't one, which is the # behavior we want this method to exhibit, so just let it bubble up. dest_repo_importer = importer_manager.get_importer(dest_repo_id) source_repo_importer = importer_manager.get_importer(source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = plugin_api.list_importer_types( dest_repo_importer['importer_type_id'])['types'] # If criteria is specified, retrieve the list of units now associate_us = None if criteria is not None: associate_us = load_associated_units(source_repo_id, criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) == 0: # Return an empty list to indicate nothing was copied return {'units_successful': []} # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units associated_unit_type_ids = calculate_associated_type_ids( source_repo_id, associate_us) unsupported_types = [ t for t in associated_unit_type_ids if t not in supported_type_ids ] if len(unsupported_types) > 0: raise exceptions.InvalidValue(['types']) # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: transfer_units = create_transfer_units(associate_us, associated_unit_type_ids) # Convert the two repos into the plugin API model transfer_dest_repo = dest_repo.to_transfer_repo() transfer_source_repo = source_repo.to_transfer_repo() # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id( dest_repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'], import_config_override) conduit = ImportUnitConduit(source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id']) try: copied_units = importer_instance.import_units(transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) unit_ids = [u.to_id_dict() for u in copied_units] return {'units_successful': unit_ids} except Exception: msg = _( 'Exception from importer [%(i)s] while importing units into repository [%(r)s]' ) msg = msg % { 'i': dest_repo_importer['importer_type_id'], 'r': dest_repo_id } logger.exception(msg) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def _validate_filters(filters): if filters is None: return None if not isinstance(filters, dict): raise pulp_exceptions.InvalidValue(['filters']) return filters
def from_client_input(cls, query): """ Parses a unit association query document and assembles a corresponding internal criteria object. Example: { "type_ids" : ["rpm"], "filters" : { "unit" : <mongo spec syntax>, "association" : <mongo spec syntax> }, "sort" : { "unit" : [ ["name", "ascending"], ["version", "descending"] ], "association" : [ ["created", "descending"] ] }, "limit" : 100, "skip" : 200, "fields" : { "unit" : ["name", "version", "arch"], "association" : ["created"] }, "remove_duplicates" : True } @param query: user-provided query details @type query: dict @return: criteria object for the unit association query @rtype: L{UnitAssociationCriteria} @raises ValueError: on an invalid value in the query """ query = copy.copy(query) type_ids = query.pop('type_ids', None) filters = query.pop('filters', None) if filters is None: association_filters = None unit_filters = None else: association_filters = _validate_filters(filters.pop('association', None)) unit_filters = _validate_filters(filters.pop('unit', None)) sort = query.pop('sort', None) if sort is None: association_sort = None unit_sort = None else: association_sort = _validate_sort(sort.pop('association', None)) unit_sort = _validate_sort(sort.pop('unit', None)) limit = _validate_limit(query.pop('limit', None)) skip = _validate_skip(query.pop('skip', None)) fields = query.pop('fields', None) if fields is None: association_fields = None unit_fields = None else: association_fields = _validate_fields(fields.pop('association', None)) unit_fields = _validate_fields(fields.pop('unit', None)) remove_duplicates = bool(query.pop('remove_duplicates', False)) # report any superfluous doc key, value pairs as errors for d in (query, filters, sort, fields): if d: raise pulp_exceptions.InvalidValue(d.keys()) # These are here for backward compatibility, in the future, these # should be removed and the corresponding association_spec and unit_spec # properties should be used if association_filters: _compile_regexs_for_not(association_filters) if unit_filters: _compile_regexs_for_not(unit_filters) return cls(type_ids=type_ids, association_filters=association_filters, unit_filters=unit_filters, association_sort=association_sort, unit_sort=unit_sort, limit=limit, skip=skip, association_fields=association_fields, unit_fields=unit_fields, remove_duplicates=remove_duplicates)
try: importer_manager.set_importer(repo_id, importer_type_id, importer_repo_plugin_config) except Exception: _logger.exception( 'Exception adding importer to repo [%s]; the repo will be deleted' % repo_id) repo.delete() raise # Add the distributors. Delete the repository if this fails. distributor_manager = manager_factory.repo_distributor_manager() for distributor in distributor_list: if not isinstance(distributor, dict): repo.delete() raise pulp_exceptions.InvalidValue(['distributor_list']) try: # Validation will occur in distributor manager. type_id = distributor.get('distributor_type_id') plugin_config = distributor.get('distributor_config') auto_publish = distributor.get('auto_publish', False) distributor_id = distributor.get('distributor_id') distributor_manager.add_distributor(repo_id, type_id, plugin_config, auto_publish, distributor_id) except Exception: _logger.exception( 'Exception adding distributor to repo [%s]; the repo will be ' 'deleted' % repo_id) repo.delete() raise
def create_and_configure_repo_group(group_id, display_name=None, description=None, repo_ids=None, notes=None, distributor_list=None): """ Create a new repository group and add distributors in a single call. This is equivalent to calling RepoGroupManager.create_repo_group and then RepoGroupDistributorManager.add_distributor for each distributor in the distributor list. :param group_id: unique id of the repository group :type group_id: str :param display_name: user-friendly name of the repository id :type display_name: str or None :param description: description of the repository group :type description: str or None :param repo_ids: the list of repository ids in this repository group :type repo_ids: list of str or None :param notes: A collection of key=value pairs :type notes: dict or None :param distributor_list: A list of dictionaries used to add distributors. The following keys are expected: from pulp.common.constants: DISTRIBUTOR_TYPE_ID_KEY, DISTRIBUTOR_CONFIG_KEY, and DISTRIBUTOR_ID_KEY, which should hold values str, dict, and str or None :type distributor_list: list of dict :return: SON representation of the repo group :rtype: bson.SON """ if distributor_list is None: distributor_list = () # Validate the distributor list before creating a repo group if not isinstance(distributor_list, (list, tuple)) or not \ all(isinstance(dist, dict) for dist in distributor_list): raise pulp_exceptions.InvalidValue(['distributor_list']) # Create the repo group using the vanilla group create method repo_group = RepoGroupManager.create_repo_group( group_id, display_name, description, repo_ids, notes) for distributor in distributor_list: try: # Attempt to add the distributor to the group. type_id = distributor.get( distributor_constants.DISTRIBUTOR_TYPE_ID_KEY) plugin_config = distributor.get( distributor_constants.DISTRIBUTOR_CONFIG_KEY) distributor_id = distributor.get( distributor_constants.DISTRIBUTOR_ID_KEY) RepoGroupDistributorManager.add_distributor( group_id, type_id, plugin_config, distributor_id) except Exception: # If an exception occurs, pass it on after cleaning up the repository group _logger.exception( 'Exception adding distributor to repo group [%s]; the group will' ' be deleted' % group_id) RepoGroupManager.delete_repo_group(group_id) raise return repo_group
def add_distributor(repo_id, distributor_type_id, repo_plugin_config, auto_publish, distributor_id=None): """ Adds an association from the given repository to a distributor. The distributor_id is unique for a given repository. If distributor_id is not specified, one will be generated. If a distributor already exists on the repo for the given ID, the existing one will be removed and replaced with the newly configured one. :param repo_id: identifies the repo :type repo_id: basestring :param distributor_type_id: must correspond to a distributor type loaded at server startup :type distributor_type_id: basestring :param repo_plugin_config: configuration the repo will use with this distributor :type repo_plugin_config: dict or None :param auto_publish: if True, this distributor will be invoked at the end of every sync :type auto_publish: bool :param distributor_id: unique ID to refer to this distributor for this repo :type distributor_id: basestring :return: distributor object :rtype: pulp.server.db.model.Distributor :raise InvalidValue: if the distributor ID is provided and unacceptable :raise exceptions.PulpDataException: if the plugin returns that the config is invalid """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) if not plugin_api.is_valid_distributor(distributor_type_id): raise exceptions.InvalidValue(['distributor_type_id']) if distributor_id is None: distributor_id = str(uuid.uuid4()) distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id) # Remove any keys whose values are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the distributor plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) config_conduit = RepoConfigConduit(distributor_type_id) transfer_repo = repo_obj.to_transfer_repo() result = distributor_instance.validate_config(transfer_repo, call_config, config_conduit) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise exceptions.PulpDataException(message) try: model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=distributor_id) delete(repo_id, distributor_id) except exceptions.MissingResource: pass # if it didn't exist, no problem distributor_instance.distributor_added(transfer_repo, call_config) distributor = model.Distributor(repo_id, distributor_id, distributor_type_id, clean_config, auto_publish) distributor.save() return distributor
def update(repo_id, dist_id, config=None, delta=None): """ Update the distributor and (re)bind any bound consumers. :param distributor: distributor to be updated :type distributor: pulp.server.db.model.Distributor :param config: A configuration dictionary for a distributor instance. The contents of this dict depends on the type of distributor. Values of None will remove they key from the config. Keys ommited from this dictionary will remain unchanged. :type config: dict :param delta: A dictionary used to change conf values for a distributor instance. This currently only supports the 'auto_publish' keyword, which should have a value of type bool :type delta: dict or None :return: result containing any errors and tasks spawned :rtype pulp.server.async.tasks.TaskResult """ repo = model.Repository.objects.get_repo_or_missing_resource(repo_id) distributor = model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=dist_id) for k, v in config.iteritems(): if v is None: distributor.config.pop(k) else: distributor.config[k] = v auto_publish = delta.get('auto_publish') if delta else None if isinstance(auto_publish, bool): distributor.auto_publish = auto_publish elif not isinstance(auto_publish, type(None)): raise exceptions.InvalidValue(['auto_publish']) # Let the distributor plugin verify the configuration distributor_instance, plugin_config = plugin_api.get_distributor_by_id( distributor.distributor_type_id) call_config = PluginCallConfiguration(plugin_config, distributor.config) transfer_repo = repo.to_transfer_repo() config_conduit = RepoConfigConduit(distributor.distributor_type_id) result = distributor_instance.validate_config(transfer_repo, call_config, config_conduit) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise exceptions.PulpDataException(message) distributor.save() unbind_errors = [] additional_tasks = [] options = {} bind_manager = managers.consumer_bind_manager() for bind in bind_manager.find_by_distributor(distributor.repo_id, distributor.distributor_id): try: report = bind_manager.bind(bind['consumer_id'], bind['repo_id'], bind['distributor_id'], bind['notify_agent'], bind['binding_config'], options) if report: additional_tasks.extend(report.spawned_tasks) except Exception, e: unbind_errors.append(e)
def associate_unit_by_id(self, repo_id, unit_type_id, unit_id, owner_type, owner_id, update_unit_count=True): """ Creates an association between the given repository and content unit. If there is already an association between the given repo and content unit where all other metadata matches the input to this method, this call has no effect. Both repo and unit must exist in the database prior to this call, however this call will not verify that for performance reasons. Care should be taken by the caller to preserve the data integrity. @param repo_id: identifies the repo @type repo_id: str @param unit_type_id: identifies the type of unit being added @type unit_type_id: str @param unit_id: uniquely identifies the unit within the given type @type unit_id: str @param owner_type: category of the caller making the association; must be one of the OWNER_* variables in this module @type owner_type: str @param owner_id: identifies the caller making the association, either the importer ID or user login @type owner_id: str @param update_unit_count: if True, updates the unit association count after the new association is made. Set this to False when doing bulk associations, and make one call to update the count at the end. defaults to True @type update_unit_count: bool @raise InvalidType: if the given owner type is not of the valid enumeration """ if owner_type not in _OWNER_TYPES: raise exceptions.InvalidValue(['owner_type']) # If the association already exists, no need to do anything else spec = {'repo_id' : repo_id, 'unit_id' : unit_id, 'unit_type_id' : unit_type_id, 'owner_type' : owner_type, 'owner_id' : owner_id,} existing_association = RepoContentUnit.get_collection().find_one(spec) if existing_association is not None: return similar_exists = False if update_unit_count: similar_exists = self.association_exists(repo_id, unit_id, unit_type_id) # Create the database entry association = RepoContentUnit(repo_id, unit_id, unit_type_id, owner_type, owner_id) RepoContentUnit.get_collection().save(association, safe=True) # update the count of associated units on the repo object if update_unit_count and not similar_exists: manager = manager_factory.repo_manager() manager.update_unit_count(repo_id, unit_type_id, 1)
def associate_from_repo(self, source_repo_id, dest_repo_id, criteria=None, import_config_override=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. @param source_repo_id: identifies the source repository @type source_repo_id: str @param dest_repo_id: identifies the destination repository @type dest_repo_id: str @param criteria: optional; if specified, will filter the units retrieved from the source repository @type criteria: L{UnitAssociationCriteria} @param import_config_override: optional config containing values to use for this import only @type import_config_override: dict @return: list of unit IDs (see pulp.plugins.model.Unit.to_id_dict) for units that were associated by this operation @rtype: list @raise MissingResource: if either of the specified repositories don't exist """ # Validation repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() source_repo = repo_query_manager.get_repository(source_repo_id) dest_repo = repo_query_manager.get_repository(dest_repo_id) # This will raise MissingResource if there isn't one, which is the # behavior we want this method to exhibit, so just let it bubble up. dest_repo_importer = importer_manager.get_importer(dest_repo_id) source_repo_importer = importer_manager.get_importer(source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = plugin_api.list_importer_types(dest_repo_importer['importer_type_id'])['types'] # If criteria is specified, retrieve the list of units now associate_us = None if criteria is not None: associate_us = load_associated_units(source_repo_id, criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) is 0: # Return an empty list to indicate nothing was copied return [] # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units associated_unit_type_ids = calculate_associated_type_ids(source_repo_id, associate_us) unsupported_types = [t for t in associated_unit_type_ids if t not in supported_type_ids] if len(unsupported_types) > 0: raise exceptions.InvalidValue(['types']) # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: transfer_units = create_transfer_units(associate_us, associated_unit_type_ids) # Convert the two repos into the plugin API model transfer_dest_repo = common_utils.to_transfer_repo(dest_repo) transfer_dest_repo.working_dir = common_utils.importer_working_dir(dest_repo_importer['importer_type_id'], dest_repo['id'], mkdir=True) transfer_source_repo = common_utils.to_transfer_repo(source_repo) transfer_source_repo.working_dir = common_utils.importer_working_dir(source_repo_importer['importer_type_id'], source_repo['id'], mkdir=True) # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id(dest_repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'], import_config_override) login = manager_factory.principal_manager().get_principal()['login'] conduit = ImportUnitConduit(source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id'], RepoContentUnit.OWNER_TYPE_USER, login) try: copied_units = importer_instance.import_units(transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) unit_ids = [u.to_id_dict() for u in copied_units] return unit_ids except Exception: _LOG.exception('Exception from importer [%s] while importing units into repository [%s]' % (dest_repo_importer['importer_type_id'], dest_repo_id)) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]