def update_distributor_config(self, repo_group_id, distributor_id,
distributor_config):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If
not, this method will raise an error and the existing configuration
will remain unchanged.
@param repo_group_id: identifies the group
@type repo_group_id: str
@param distributor_id: identifies the distributor on the group
@type distributor_id: str
@param distributor_config: new configuration values to use
@type distributor_config: dict
@return: the updated distributor
@rtype: dict
@raise MissingResource: if the given group or distributor do not exist
@raise PulpDataException: if the plugin indicates the new configuration
is invalid
"""
# Validation - calls will raise MissingResource
group_manager = manager_factory.repo_group_query_manager()
group = group_manager.get_group(repo_group_id)
distributor = self.get_distributor(repo_group_id, distributor_id)
distributor_type_id = distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id(
distributor_type_id)
# Resolve the requested changes into the existing config
merged_config = process_update_config(distributor['config'],
distributor_config)
# Request the distributor validate the new configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_group = common_utils.to_transfer_repo_group(group)
transfer_group.working_dir = common_utils.group_distributor_working_dir(
distributor_type_id, repo_group_id)
transfer_related_groups = related_groups(distributor_type_id,
omit_group_id=repo_group_id)
# Request the plugin validate the configuration
try:
is_valid, message = distributor_instance.validate_config(
transfer_group, call_config, transfer_related_groups)
if not is_valid:
raise PulpDataException(message)
except Exception, e:
_LOG.exception(
'Exception received from distributor [%s] while validating config'
% distributor_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def update_distributor_config(repo_group_id, distributor_id,
distributor_config):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If
not, this method will raise an error and the existing configuration
will remain unchanged.
:param repo_group_id: identifies the group
:type repo_group_id: str
:param distributor_id: identifies the distributor on the group
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given group or distributor do not exist
:raise PulpDataException: if the plugin indicates the new configuration is invalid
"""
# Validation - calls will raise MissingResource
group_manager = manager_factory.repo_group_query_manager()
group = group_manager.get_group(repo_group_id)
distributor = RepoGroupDistributorManager.get_distributor(
repo_group_id, distributor_id)
distributor_type_id = distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id(
distributor_type_id)
# Resolve the requested changes into the existing config
merged_config = process_update_config(distributor['config'],
distributor_config)
# Request the distributor validate the new configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_group = common_utils.to_transfer_repo_group(group)
config_conduit = RepoConfigConduit(distributor_type_id)
# Request the plugin validate the configuration
try:
is_valid, message = distributor_instance.validate_config(
transfer_group, call_config, config_conduit)
if not is_valid:
raise PulpDataException(message)
except Exception, e:
msg = _(
'Exception received from distributor [%(d)s] while validating config'
)
msg = msg % {'d': distributor_type_id}
_logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def is_valid_upload(repo_id, unit_type_id):
"""
Checks that the repository is configured to handle an upload request
for the given unit type ID. This should be called prior to beginning
the upload to prevent a wasted effort in the bits uploading.
:param repo_id: identifies the repo into which the unit is being uploaded
:param unit_type_id: type of unit being uploaded
:return: true if the repository can attempt to handle the unit
:rtype: bool
:raise MissingResource: if the repository or its importer do not exist
"""
importer_manager = manager_factory.repo_importer_manager()
# Will raise an appropriate exception if it cannot be found
repo_importer = importer_manager.get_importer(repo_id)
# Make sure the importer on the repo can support the indicated type
importer_types = plugin_api.list_importer_types(repo_importer['importer_type_id'])['types']
if unit_type_id not in importer_types:
raise PulpDataException('Invalid unit type for repository')
return True
def is_last_super_user(self, login):
"""
Check to see if a user is the last super user
@type user: str
@param user: login of user to check
@rtype: bool
@return: True if the user is the last super user, False otherwise
@raise PulpDataException: if no super users are found
"""
user = User.get_collection().find_one({'login': login})
role_manager = factory.role_manager()
if SUPER_USER_ROLE not in user['roles']:
return False
users = self.find_users_belonging_to_role(SUPER_USER_ROLE)
if not users:
raise PulpDataException(_('no super users defined'))
if len(users) >= 2:
return False
return users[0]['_id'] == user['_id'] # this should be True
def update_permission(resource_uri, delta):
"""
Updates a permission object.
:param resource_uri: identifies the resource URI of the permission being deleted
:type resource_uri: str
:param delta: A dict containing update keywords.
:type delta: dict
:raises MissingResource: if the permission does not exist
:raises PulpDataException: if some usupported keys were specified
"""
# Check whether the permission exists
found = Permission.get_collection().find_one({'resource': resource_uri})
if found is None:
raise MissingResource(resource_uri)
for key, value in delta.items():
# simple changes
if key in ('users',):
found[key] = value
continue
# unsupported
raise PulpDataException(_("Update Keyword [%s] is not supported" % key))
Permission.get_collection().save(found, safe=True)
def add_permissions_to_role(self, role_id, resource, operations):
"""
Add permissions to a role.
@type role_id: str
@param role_id: role identifier
@type resource: str
@param resource: resource path to grant permissions to
@type operations: list of allowed operations being granted
@param operations: list or tuple
@raise MissingResource: if the given role does not exist
"""
if role_id == self.super_user_role:
raise PulpDataException(_('super-users role cannot be changed'))
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
current_ops = role['permissions'].setdefault(resource, [])
for o in operations:
if o in current_ops:
continue
current_ops.append(o)
users = factory.user_query_manager().find_users_belonging_to_role(
role_id)
for user in users:
factory.permission_manager().grant(resource, user['login'],
operations)
Role.get_collection().save(role, safe=True)
def delete_user(login):
"""
Deletes the given user. Deletion of last superuser is not permitted.
@param login: identifies the user being deleted
@type login: str
@raise MissingResource: if the given user does not exist
@raise InvalidValue: if login value is invalid
"""
# Raise exception if login is invalid
if login is None or invalid_type(login, basestring):
raise InvalidValue(['login'])
# Check whether user exists
found = User.get_collection().find_one({'login': login})
if found is None:
raise MissingResource(login)
# Make sure user is not the last super user
if factory.user_query_manager().is_last_super_user(login):
raise PulpDataException(_("The last superuser [%s] cannot be deleted" % login))
# Revoke all permissions from the user
permission_manager = factory.permission_manager()
permission_manager.revoke_all_permissions_from_user(login)
User.get_collection().remove({'login': login})
def update_permission(self, resource_uri, delta):
"""
Updates a permission object.
@param resource_uri: identifies the resource URI of the permission being deleted
@type resource_uri: str
@param delta: A dict containing update keywords.
@type delta: dict
@return: The updated object
@rtype: dict
"""
# Check whether the permission exists
found = Permission.get_collection().find_one(
{'resource': resource_uri})
if found is None:
raise MissingResource(resource_uri)
for key, value in delta.items():
# simple changes
if key in ('users', ):
found[key] = value
continue
# unsupported
raise PulpDataException(
_("Update Keyword [%s] is not supported" % key))
Permission.get_collection().save(found, safe=True)
def publish_repo(self, transfer_repo, publish_conduit, config):
"""
Export a yum repository to a given directory, or to ISO
:param transfer_repo: metadata describing the repository
:type transfer_repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and
# currently, validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
_logger.info('Starting export of [%s]' % transfer_repo.id)
self._publisher = ExportRepoPublisher(transfer_repo, publish_conduit,
config,
ids.TYPE_ID_DISTRIBUTOR_EXPORT)
return self._publisher.process_lifecycle()
def publish_group(self, repo_group, publish_conduit, config):
"""
Publishes the given repository group.
:param repo_group: metadata describing the repository group
:type repo_group: pulp.plugins.model.RepositoryGroup
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options,
# and currently,
# validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
# raises a PulpCodedException if all units are not downloaded
self.ensure_all_units_downloaded(repo_group)
_logger.info(
'Beginning export of the following repository group: [%s]' %
repo_group.id)
self._publisher = ExportRepoGroupPublisher(
repo_group, publish_conduit, config,
ids.TYPE_ID_DISTRIBUTOR_GROUP_EXPORT)
return self._publisher.process_lifecycle()
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict
:raise MissingResource: if repo_id does not represent a valid repo
:raise InvalidImporterConfiguration: if the importer cannot be initialized for the given
repo
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise InvalidValue(['importer_type_id'])
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id)
try:
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
logger.exception(
'Exception received from importer [%s] while validating config' % importer_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def remove_permissions_from_role(role_id, resource, operations):
"""
Remove permissions from a role.
:param role_id: role identifier
:type role_id: str
:param resource: resource path to revoke permissions from
:type resource: str
:param operations: list or tuple
:type operations: list of allowed operations being revoked
:raise InvalidValue: if some params are invalid
:raise PulpDataException: if role is a superuser role
"""
if role_id == SUPER_USER_ROLE:
raise PulpDataException(_('super-users role cannot be changed'))
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise InvalidValue(['role_id'])
resource_permission = {}
current_ops = []
for item in role['permissions']:
if item['resource'] == resource:
resource_permission = item
current_ops = resource_permission['permission']
if not current_ops:
return
for o in operations:
if o not in current_ops:
continue
current_ops.remove(o)
users = factory.user_query_manager().find_users_belonging_to_role(
role_id)
for user in users:
other_roles = factory.role_query_manager().get_other_roles(
role, user['roles'])
user_ops = _operations_not_granted_by_roles(
resource, operations, other_roles)
factory.permission_manager().revoke(resource, user['login'],
user_ops)
# in no more allowed operations, remove the resource
if not current_ops:
role['permissions'].remove(resource_permission)
Role.get_collection().save(role, safe=True)
def add_permissions_to_role(role_id, resource, operations):
"""
Add permissions to a role.
:param role_id: role identifier
:type role_id: str
:param resource: resource path to grant permissions to
:type resource: str
:param operations: list or tuple
:type operations: list of allowed operations being granted
:raise InvalidValue: if some params are invalid
:raise PulpDataException: if role is a superuser role
"""
if role_id == SUPER_USER_ROLE:
raise PulpDataException(_('super-users role cannot be changed'))
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise InvalidValue(['role_id'])
if not role['permissions']:
role['permissions'] = []
resource_permission = {}
current_ops = []
for item in role['permissions']:
if item['resource'] == resource:
resource_permission = item
current_ops = resource_permission['permission']
if not resource_permission:
resource_permission = dict(resource=resource,
permission=current_ops)
role['permissions'].append(resource_permission)
for o in operations:
if o in current_ops:
continue
current_ops.append(o)
users = factory.user_query_manager().find_users_belonging_to_role(
role_id)
for user in users:
factory.permission_manager().grant(resource, user['login'],
operations)
Role.get_collection().save(role, safe=True)
def remove_permissions_from_role(self, role_id, resource, operations):
"""
Remove permissions from a role.
@type role_id: str
@param role_id: role identifier
@type resource: str
@param resource: resource path to revoke permissions from
@type operations: list of allowed operations being revoked
@param operations: list or tuple
@raise MissingResource: if the given role does not exist
"""
if role_id == self.super_user_role:
raise PulpDataException(_('super-users role cannot be changed'))
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
current_ops = role['permissions'].get(resource, [])
if not current_ops:
return
for o in operations:
if o not in current_ops:
continue
current_ops.remove(o)
users = factory.user_query_manager().find_users_belonging_to_role(
role_id)
for user in users:
other_roles = factory.role_query_manager().get_other_roles(
role, user['roles'])
user_ops = _operations_not_granted_by_roles(
resource, operations, other_roles)
factory.permission_manager().revoke(resource, user['login'],
user_ops)
# in no more allowed operations, remove the resource
if not current_ops:
del role['permissions'][resource]
Role.get_collection().save(role, safe=True)
def delete_role(role_id):
"""
Deletes the given role. This has the side-effect of revoking any permissions granted
to the role from the users in the role, unless those permissions are also granted
through another role the user is a memeber of.
:param role_id: identifies the role being deleted
:type role_id: str
:raise InvalidValue: if any of the fields are unacceptable
:raise MissingResource: if the given role does not exist
:raise PulpDataException: if role is a superuser role
"""
# Raise exception if role id is invalid
if role_id is None or not isinstance(role_id, basestring):
raise InvalidValue(['role_id'])
# Check whether role exists
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
# Make sure role is not a superuser role
if role_id == SUPER_USER_ROLE:
raise PulpDataException(_('Role %s cannot be changed') % role_id)
# Remove respective roles from users
users = factory.user_query_manager().find_users_belonging_to_role(
role_id)
for item in role['permissions']:
for user in users:
other_roles = factory.role_query_manager().get_other_roles(
role, user['roles'])
user_ops = _operations_not_granted_by_roles(
item['resource'], item['permission'], other_roles)
factory.permission_manager().revoke(item['resource'],
user['login'], user_ops)
for user in users:
user['roles'].remove(role_id)
factory.user_manager().update_user(user['login'],
Delta(user, 'roles'))
Role.get_collection().remove({'id': role_id}, safe=True)
def remove_user_from_role(self, role_id, login):
"""
Remove a user from a role. This has the side-effect of revoking all the
permissions granted to the role from the user, unless the permissions are
also granted by another role.
@type role_id: str
@param role_id: role identifier
@type login: str
@param login: name of user
@rtype: bool
@return: True on success
@raise MissingResource: if the given role or user does not exist
"""
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
user = User.get_collection().find_one({'login': login})
if user is None:
raise MissingResource(login)
if role_id == self.super_user_role and factory.user_query_manager(
).is_last_super_user(login):
raise PulpDataException(
_('%s cannot be empty, and %s is the last member') %
(self.super_user_role, login))
if role_id not in user['roles']:
return
user['roles'].remove(role_id)
User.get_collection().save(user, safe=True)
for resource, operations in role['permissions'].items():
other_roles = factory.role_query_manager().get_other_roles(
role, user['roles'])
user_ops = _operations_not_granted_by_roles(
resource, operations, other_roles)
factory.permission_manager().revoke(resource, login, user_ops)
def remove_user_from_role(role_id, login):
"""
Remove a user from a role. This has the side-effect of revoking all the
permissions granted to the role from the user, unless the permissions are
also granted by another role.
:param role_id: role identifier
:type role_id: str
:param login: name of user
:type login: str
:raise MissingResource: if the given role or user does not exist
"""
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
user = User.get_collection().find_one({'login': login})
if user is None:
raise MissingResource(login)
if role_id == SUPER_USER_ROLE and factory.user_query_manager(
).is_last_super_user(login):
raise PulpDataException(
_('%(role)s cannot be empty, and %(login)s is the last member')
% {
'role': SUPER_USER_ROLE,
'login': login
})
if role_id not in user['roles']:
return
user['roles'].remove(role_id)
User.get_collection().save(user, safe=True)
for item in role['permissions']:
other_roles = factory.role_query_manager().get_other_roles(
role, user['roles'])
user_ops = _operations_not_granted_by_roles(
item['resource'], item['permission'], other_roles)
factory.permission_manager().revoke(item['resource'], login,
user_ops)
def update_role(self, role_id, delta):
"""
Updates a role object.
@param role_id: The role identifier.
@type role_id: str
@param delta: A dict containing update keywords.
@type delta: dict
@return: The updated object
@rtype: dict
@raise MissingResource: if the given role does not exist
@raise PulpDataException: if update keyword is not supported
"""
delta.pop('id', None)
role = Role.get_collection().find_one({'id': role_id})
if role is None:
raise MissingResource(role_id)
for key, value in delta.items():
# simple changes
if key in (
'display_name',
'description',
'permissions',
):
role[key] = value
continue
# unsupported
raise PulpDataException(
_("Update Keyword [%s] is not supported" % key))
Role.get_collection().save(role, safe=True)
# Retrieve the user to return the SON object
updated = Role.get_collection().find_one({'id': role_id})
return updated
def add_distributor(repo_group_id,
distributor_type_id,
group_plugin_config,
distributor_id=None):
"""
Adds an association from the given repository group to a distributor.
The assocation will be tracked through the distributor_id; each
distributor on a given group must have a unique ID. If this is not
specified, one will be generated. If a distributor already exists on the
group with a given ID, the existing one will be removed and replaced
with the newly configured one.
@param repo_group_id: identifies the repo group
@type repo_group_id: str
@param distributor_type_id: type of distributor being added; must reference
one of the installed group distributors
@type distributor_type_id: str
@param group_plugin_config: config to use for the distributor for this group alone
@type group_plugin_config: dict
@param distributor_id: if specified, the newly added distributor will be
referenced by this value and the group id; if omitted one will
be generated
@type distributor_id: str
@return: database representation of the added distributor
@rtype: dict
@raise MissingResource: if the group doesn't exist
@raise InvalidValue: if a distributor ID is provided and is not valid
@raise PulpDataException: if the plugin indicates the config is invalid
@raise PulpExecutionException: if the plugin raises an exception while
initializing the newly added distributor
"""
distributor_coll = RepoGroupDistributor.get_collection()
query_manager = manager_factory.repo_group_query_manager()
# Validation
group = query_manager.get_group(
repo_group_id) # will raise MissingResource
if not plugin_api.is_valid_group_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for the distributor on this repo
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id(
distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
clean_config = None
if group_plugin_config is not None:
clean_config = dict([(k, v)
for k, v in group_plugin_config.items()
if v is not None])
# Let the plugin validate the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_group = common_utils.to_transfer_repo_group(group)
config_conduit = RepoConfigConduit(distributor_type_id)
# Request the plugin validate the configuration
try:
is_valid, message = distributor_instance.validate_config(
transfer_group, call_config, config_conduit)
if not is_valid:
raise PulpDataException(message)
except Exception, e:
msg = _(
'Exception received from distributor [%(d)s] while validating config'
)
msg = msg % {'d': distributor_type_id}
_logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def update_distributor_config(repo_id,
distributor_id,
distributor_config,
auto_publish=None):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_id: identifies the distributor on the repo
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:param auto_publish: If true, this distributor is used automatically during a sync operation
:type auto_publish: bool
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given repo or distributor doesn't exist
:raise PulpDataException: if the plugin rejects the given changes
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Input Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repository=repo_id)
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
if repo_distributor is None:
raise MissingResource(distributor=distributor_id)
distributor_type_id = repo_distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(
distributor_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_distributor['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [
k for k in distributor_config if distributor_config[k] is None
]
for key in unset_property_names:
merged_config.pop(key, None)
distributor_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(distributor_config)
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
distributor_type_id, repo_id)
config_conduit = RepoConfigConduit(distributor_type_id)
try:
result = distributor_instance.validate_config(
transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
msg = _(
'Exception raised from distributor [%(d)s] while validating config for repo '
'[%(r)s]')
msg = msg % {'d': distributor_type_id, 'r': repo_id}
logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def _invoke_plugin(call, *args, **kwargs):
try:
return call(*args, **kwargs)
except InvalidUnitsRequested, e:
trace = sys.exc_info()[2]
raise PulpDataException(e.message), None, trace
def update_importer_config(repo_id, importer_config):
"""
Attempts to update the saved configuration for the given repo's importer.
The importer will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_config: new configuration values to use for this repo
:type importer_config: dict
:raise MissingResource: if the given repo does not exist
:raise MissingResource: if the given repo does not have an importer
:raise InvalidConfiguration: if the plugin indicates the given configuration is invalid
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Input Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_importer = importer_coll.find_one({'repo_id': repo_id})
if repo_importer is None:
raise MissingResource(repo_id)
importer_type_id = repo_importer['importer_type_id']
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_importer['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [
k for k in importer_config if importer_config[k] is None
]
for key in unset_property_names:
merged_config.pop(key, None)
importer_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(importer_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = common_utils.to_transfer_repo(repo)
try:
result = importer_instance.validate_config(transfer_repo,
call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
msg = _(
'Exception received from importer [%(i)s] while validating config for repo '
'[%(r)s]')
msg = msg % {'i': importer_type_id, 'r': repo_id}
_logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
class RepoImporterManager(object):
def get_importer(self, repo_id):
"""
Returns metadata about an importer associated with the given repo.
@return: key-value pairs describing the importer in use
@rtype: dict
@raise MissingResource: if the repo does not exist or has no importer associated
"""
importer = RepoImporter.get_collection().find_one({'repo_id': repo_id})
if importer is None:
raise MissingResource(repository=repo_id)
return importer
def get_importers(self, repo_id):
"""
Returns a list of all importers associated with the given repo.
@return: list of key-value pairs describing the importers in use; empty
list if the repo has no importers
@rtype: list of dict
@raise MissingResource: if the given repo doesn't exist
"""
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
importers = RepoImporter.get_collection().find({'repo_id': repo_id})
return list(importers)
@staticmethod
def find_by_repo_list(repo_id_list):
"""
Returns serialized versions of all importers for given repos. Any
IDs that do not refer to valid repos are ignored and will not
raise an error.
@param repo_id_list: list of importer IDs to fetch
@type repo_id_list: list of str
@return: list of serialized importers
@rtype: list of dict
"""
spec = {'repo_id': {'$in': repo_id_list}}
projection = {'scratchpad': 0}
importers = RepoImporter.get_collection().find(spec, projection)
return list(importers)
@staticmethod
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict
:raise MissingResource: if repo_id does not represent a valid repo
:raise InvalidImporterConfiguration: if the importer cannot be initialized for the given
repo
"""
RepoImporterManager.validate_importer_config(repo_id, importer_type_id,
repo_plugin_config)
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
repo = repo_coll.find_one({'id': repo_id})
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items()
if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
# Remove old importer if one exists
try:
RepoImporterManager.remove_importer(repo_id)
except MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the repository
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_logger.exception(
'Error initializing importer [%s] for repo [%s]' %
(importer_type_id, repo_id))
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
importer_id = importer_type_id # use the importer name as its repo ID
importer = RepoImporter(repo_id, importer_id, importer_type_id,
clean_config)
importer_coll.save(importer, safe=True)
return importer
@staticmethod
def validate_importer_config(repo_id, importer_type_id, importer_config):
"""
Validate an importer configuration. This validates that the repository and importer type
exist as these are both required to validate the configuration.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param importer_config: configuration values for the importer; may be None
:type importer_config: dict
"""
repo_coll = Repo.get_collection()
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise PulpCodedValidationException(
error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if importer_config is not None:
clean_config = dict([(k, v) for k, v in importer_config.items()
if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpCodedValidationException(validation_errors=message)
@staticmethod
def remove_importer(repo_id):
"""
Removes an importer from a repository.
:param repo_id: identifies the repo
:type repo_id: str
:raise MissingResource: if the given repo does not exist
:raise MissingResource: if the given repo does not have an importer
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_importer = importer_coll.find_one({'repo_id': repo_id})
if repo_importer is None:
raise MissingResource(repo_id)
# remove schedules
RepoSyncScheduleManager().delete_by_importer_id(
repo_id, repo_importer['id'])
# Call the importer's cleanup method
importer_type_id = repo_importer['importer_type_id']
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
call_config = PluginCallConfiguration(plugin_config,
repo_importer['config'])
transfer_repo = common_utils.to_transfer_repo(repo)
importer_instance.importer_removed(transfer_repo, call_config)
# Update the database to reflect the removal
importer_coll.remove({'repo_id': repo_id}, safe=True)
@staticmethod
def update_importer_config(repo_id, importer_config):
"""
Attempts to update the saved configuration for the given repo's importer.
The importer will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_config: new configuration values to use for this repo
:type importer_config: dict
:raise MissingResource: if the given repo does not exist
:raise MissingResource: if the given repo does not have an importer
:raise InvalidConfiguration: if the plugin indicates the given configuration is invalid
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Input Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_importer = importer_coll.find_one({'repo_id': repo_id})
if repo_importer is None:
raise MissingResource(repo_id)
importer_type_id = repo_importer['importer_type_id']
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_importer['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [
k for k in importer_config if importer_config[k] is None
]
for key in unset_property_names:
merged_config.pop(key, None)
importer_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(importer_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = common_utils.to_transfer_repo(repo)
try:
result = importer_instance.validate_config(transfer_repo,
call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
msg = _(
'Exception received from importer [%(i)s] while validating config for repo '
'[%(r)s]')
msg = msg % {'i': importer_type_id, 'r': repo_id}
_logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
if not valid_config:
raise PulpDataException(message)
# If we got this far, the new config is valid, so update the database
repo_importer['config'] = merged_config
importer_coll.save(repo_importer, safe=True)
serializer = serializers.ImporterSerializer(repo_importer)
return serializer.data
class RepoDistributorManager(object):
def get_distributor(self, repo_id, distributor_id):
"""
Returns an individual distributor on the given repo.
@param repo_id: identifies the repo
@type repo_id: str
@param distributor_id: identifies the distributor
@type distributor_id: str
@return: key-value pairs describing the distributor
@rtype: dict
@raise MissingResource: if either the repo doesn't exist or there is no
distributor with the given ID
"""
distributor = RepoDistributor.get_collection().find_one({
'repo_id':
repo_id,
'id':
distributor_id
})
if distributor is None:
raise MissingResource(distributor=distributor_id)
return distributor
def get_distributors(self, repo_id):
"""
Returns all distributors on the given repo.
@param repo_id: identifies the repo
@type repo_id: str
@return: list of key-value pairs describing the distributors; empty list
if there are none for the given repo
@rtype: list, None
@raise MissingResource: if the given repo doesn't exist
"""
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repository=repo_id)
distributors = list(RepoDistributor.get_collection().find(
{'repo_id': repo_id}))
return distributors
@staticmethod
def find_by_repo_list(repo_id_list):
"""
Returns serialized versions of all distributors for given repos. Any
IDs that do not refer to valid repos are ignored and will not
raise an error.
@param repo_id_list: list of distributor IDs to fetch
@type repo_id_list: list of str
@return: list of serialized distributors
@rtype: list of dict
"""
spec = {'repo_id': {'$in': repo_id_list}}
projection = {'scratchpad': 0}
return list(RepoDistributor.get_collection().find(spec, projection))
@staticmethod
def add_distributor(repo_id,
distributor_type_id,
repo_plugin_config,
auto_publish,
distributor_id=None):
"""
Adds an association from the given repository to a distributor. The
association will be tracked through the distributor_id; each distributor
on a given repository must have a unique ID. If this is not specified,
one will be generated. If a distributor already exists on the repo for
the given ID, the existing one will be removed and replaced with the
newly configured one.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_type_id: identifies the distributor; must correspond to a
distributor loaded at server startup
:type distributor_type_id: str
:param repo_plugin_config: configuration the repo will use with this
distributor; may be None
:type repo_plugin_config: dict
:param auto_publish: if true, this distributor will be invoked at the end
of every sync
:type auto_publish: bool
:param distributor_id: unique ID to refer to this distributor for this repo
:type distributor_id: str
:return: ID assigned to the distributor (only valid in
conjunction with the repo)
:raise MissingResource: if the given repo_id does not refer to a valid repo
:raise InvalidValue: if the distributor ID is provided and unacceptable
:raise InvalidDistributorConfiguration: if the distributor plugin does not accept the given
configuration
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repository=repo_id)
if not plugin_api.is_valid_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for this distributor on this repo; will be
# unique for all distributors on this repository but not globally
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(
distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items()
if v is not None])
else:
clean_config = None
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
distributor_type_id, repo_id)
config_conduit = RepoConfigConduit(distributor_type_id)
try:
result = distributor_instance.validate_config(
transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
logger.exception(
'Exception received from distributor [%s] while validating config'
% distributor_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
if not valid_config:
raise PulpDataException(message)
# Remove the old distributor if it exists
try:
RepoDistributorManager.remove_distributor(repo_id, distributor_id)
except MissingResource:
pass # if it didn't exist, no problem
# Let the distributor plugin initialize the repository
try:
distributor_instance.distributor_added(transfer_repo, call_config)
except Exception:
logger.exception(
'Error initializing distributor [%s] for repo [%s]' %
(distributor_type_id, repo_id))
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
distributor = RepoDistributor(repo_id, distributor_id,
distributor_type_id, clean_config,
auto_publish)
distributor_coll.save(distributor, safe=True)
return distributor
def update_distributor_config(repo_id, distributor_id, distributor_config, auto_publish=None):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_id: identifies the distributor on the repo
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:param auto_publish: If true, this distributor is used automatically during a sync operation
:type auto_publish: bool
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given repo or distributor doesn't exist
:raise PulpDataException: if the plugin rejects the given changes
"""
distributor_coll = RepoDistributor.get_collection()
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
repo_distributor = distributor_coll.find_one({'repo_id': repo_id, 'id': distributor_id})
if repo_distributor is None:
raise MissingResource(distributor=distributor_id)
distributor_type_id = repo_distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_distributor['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [k for k in distributor_config if distributor_config[k] is None]
for key in unset_property_names:
merged_config.pop(key, None)
distributor_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(distributor_config)
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = repo_obj.to_transfer_repo()
config_conduit = RepoConfigConduit(distributor_type_id)
result = distributor_instance.validate_config(transfer_repo, call_config,
config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpDataException(message)
# Confirm that the auto_publish value is sane before updating the value, if it exists
if auto_publish is not None:
if isinstance(auto_publish, bool):
repo_distributor['auto_publish'] = auto_publish
else:
raise InvalidValue(['auto_publish'])
# If we got this far, the new config is valid, so update the database
repo_distributor['config'] = merged_config
distributor_coll.save(repo_distributor, safe=True)
return repo_distributor
def add_distributor(repo_id, distributor_type_id, repo_plugin_config,
auto_publish, distributor_id=None):
"""
Adds an association from the given repository to a distributor. The
association will be tracked through the distributor_id; each distributor
on a given repository must have a unique ID. If this is not specified,
one will be generated. If a distributor already exists on the repo for
the given ID, the existing one will be removed and replaced with the
newly configured one.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_type_id: identifies the distributor; must correspond to a
distributor loaded at server startup
:type distributor_type_id: str
:param repo_plugin_config: configuration the repo will use with this
distributor; may be None
:type repo_plugin_config: dict
:param auto_publish: if true, this distributor will be invoked at the end
of every sync
:type auto_publish: bool
:param distributor_id: unique ID to refer to this distributor for this repo
:type distributor_id: str
:return: ID assigned to the distributor (only valid in
conjunction with the repo)
:raise MissingResource: if the given repo_id does not refer to a valid repo
:raise InvalidValue: if the distributor ID is provided and unacceptable
:raise InvalidDistributorConfiguration: if the distributor plugin does not accept the given
configuration
"""
distributor_coll = RepoDistributor.get_collection()
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for this distributor on this repo; will be
# unique for all distributors on this repository but not globally
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
config_conduit = RepoConfigConduit(distributor_type_id)
transfer_repo = repo_obj.to_transfer_repo()
result = distributor_instance.validate_config(transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpDataException(message)
# Remove the old distributor if it exists
try:
RepoDistributorManager.remove_distributor(repo_id, distributor_id)
except MissingResource:
pass # if it didn't exist, no problem
# Let the distributor plugin initialize the repository
try:
distributor_instance.distributor_added(transfer_repo, call_config)
except Exception:
msg = _('Error initializing distributor [%(d)s] for repo [%(r)s]')
msg = msg % {'d': distributor_type_id, 'r': repo_id}
_logger.exception(msg)
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
distributor = RepoDistributor(repo_id, distributor_id, distributor_type_id, clean_config,
auto_publish)
distributor_coll.save(distributor, safe=True)
return distributor
def __invoke_plugin(self, call, *args, **kwargs):
try:
return call(*args, **kwargs)
except InvalidUnitsRequested, e:
raise PulpDataException(e.units, e.message)
class RepoImporterManager(object):
def get_importer(self, repo_id):
"""
Returns metadata about an importer associated with the given repo.
@return: key-value pairs describing the importer in use
@rtype: dict
@raise MissingResource: if the repo does not exist or has no importer associated
"""
importer = RepoImporter.get_collection().find_one({'repo_id': repo_id})
if importer is None:
raise MissingResource(repository=repo_id)
return importer
def get_importers(self, repo_id):
"""
Returns a list of all importers associated with the given repo.
@return: list of key-value pairs describing the importers in use; empty
list if the repo has no importers
@rtype: list of dict
@raise MissingResource: if the given repo doesn't exist
"""
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
importers = list(RepoImporter.get_collection().find(
{'repo_id': repo_id}))
return importers
@staticmethod
def find_by_repo_list(repo_id_list):
"""
Returns serialized versions of all importers for given repos. Any
IDs that do not refer to valid repos are ignored and will not
raise an error.
@param repo_id_list: list of importer IDs to fetch
@type repo_id_list: list of str
@return: list of serialized importers
@rtype: list of dict
"""
spec = {'repo_id': {'$in': repo_id_list}}
projection = {'scratchpad': 0}
importers = list(RepoImporter.get_collection().find(spec, projection))
# Process any scheduled syncs and get schedule details using schedule id
for importer in importers:
scheduled_sync_ids = importer.get('scheduled_syncs', None)
if scheduled_sync_ids is not None:
scheduled_sync_details = list(
ScheduledCall.get_collection().find(
{"id": {
"$in": scheduled_sync_ids
}}))
importer['scheduled_syncs'] = [
s["schedule"] for s in scheduled_sync_details
]
return importers
def set_importer(self, repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
@param repo_id: identifies the repo
@type repo_id; str
@param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
@type importer_type_id: str
@param repo_plugin_config: configuration values for the importer; may be None
@type repo_plugin_config: dict
@raise MissingResource: if repo_id does not represent a valid repo
@raise InvalidImporterConfiguration: if the importer cannot be
initialized for the given repo
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise InvalidValue(['importer_type_id'])
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items()
if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(
importer_type_id, repo_id)
query_manager = manager_factory.repo_query_manager()
related_repos = query_manager.find_with_importer_type(importer_type_id)
transfer_related_repos = []
for r in related_repos:
all_configs = [d['config'] for d in r['importers']]
trr = common_utils.to_related_repo(r, all_configs)
transfer_related_repos.append(trr)
try:
result = importer_instance.validate_config(transfer_repo,
call_config,
transfer_related_repos)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
_LOG.exception(
'Exception received from importer [%s] while validating config'
% importer_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
if not valid_config:
raise PulpDataException(message)
# Remove old importer if one exists
try:
self.remove_importer(repo_id)
except MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the repository
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_LOG.exception('Error initializing importer [%s] for repo [%s]' %
(importer_type_id, repo_id))
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
importer_id = importer_type_id # use the importer name as its repo ID
importer = RepoImporter(repo_id, importer_id, importer_type_id,
clean_config)
importer_coll.save(importer, safe=True)
return importer
def update_importer_config(self, repo_id, importer_config):
"""
Attempts to update the saved configuration for the given repo's importer.
The importer will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
@param repo_id: identifies the repo
@type repo_id: str
@param importer_config: new configuration values to use for this repo
@type importer_config: dict
@raise MissingResource: if the given repo does not exist
@raise MissingResource: if the given repo does not have an importer
@raise InvalidConfiguration: if the plugin indicates the given
configuration is invalid
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Input Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_importer = importer_coll.find_one({'repo_id': repo_id})
if repo_importer is None:
raise MissingResource(repo_id)
importer_type_id = repo_importer['importer_type_id']
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_importer['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [
k for k in importer_config if importer_config[k] is None
]
for key in unset_property_names:
merged_config.pop(key, None)
importer_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(importer_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(
importer_type_id, repo_id)
query_manager = manager_factory.repo_query_manager()
related_repos = query_manager.find_with_importer_type(importer_type_id)
transfer_related_repos = []
for r in related_repos:
# Don't include the repo being updated in this list
if r['id'] == repo_id:
continue
all_configs = [d['config'] for d in r['importers']]
trr = common_utils.to_related_repo(r, all_configs)
transfer_related_repos.append(trr)
try:
result = importer_instance.validate_config(transfer_repo,
call_config,
transfer_related_repos)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
_LOG.exception(
'Exception received from importer [%s] while validating config for repo [%s]'
% (importer_type_id, repo_id))
raise PulpDataException(e.args), None, sys.exc_info()[2]
transfer_related_repos)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
_LOG.exception(
'Exception received from importer [%s] while validating config for repo [%s]'
% (importer_type_id, repo_id))
raise PulpDataException(e.args), None, sys.exc_info()[2]
if not valid_config:
raise PulpDataException(message)
# If we got this far, the new config is valid, so update the database
repo_importer['config'] = merged_config
importer_coll.save(repo_importer, safe=True)
return repo_importer
def get_importer_scratchpad(self, repo_id):
"""
Returns the contents of the importer's scratchpad for the given repo.
If there is no importer or the scratchpad has not been set, None is
returned.
@param repo_id: identifies the repo
@type repo_id: str
