def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True,
distributor_id='dist-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor(
'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True,
distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor(
'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True, distributor_id='dist-5')
self.repo_manager.create_repo('repo-5')
self.distributor_manager.add_distributor(
'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True,
distributor_id='dist-1')
self.repo_manager.create_repo('repo-6')
self.distributor_manager.add_distributor(
'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"},
True, distributor_id='dist-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/c"},
True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor('repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"},
True, distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor('repo-4', 'mock-distributor', {"relative_url": "/repo-5"},
True, distributor_id='dist-5')
self.conduit = RepoConfigConduit('rpm')
def test_validate_config(self, *mock_methods):
config_kwargs = {
'http': True,
'https': True,
'relative_url': None,
'auth_ca': 'CA',
'auth_cert': 'CERT',
'checksum_type': 'sha256',
'http_publish_dir': '/http/path/',
'https_publish_dir': 'https/path/',
'protected': True,
'skip': {
'drpms': 1
},
'skip_pkg_tags': True,
'generate_sqlite': False
}
repo = Repository('test')
config = self._generate_call_config(**config_kwargs)
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
valid, reasons = configuration.validate_config(repo, config, conduit)
for mock_method in mock_methods:
self.assertEqual(mock_method.call_count, 1)
self.assertTrue(valid)
self.assertEqual(reasons, None)
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
with mock.patch('pulp.server.controllers.distributor.model.Repository.objects'):
# Populate the database with a repo with units
dist_controller.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True,
distributor_id='dist-1')
dist_controller.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True,
distributor_id='dist-2')
dist_controller.add_distributor(
'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True,
distributor_id='dist-3')
dist_controller.add_distributor(
'repo-3', 'mock-distributor', {}, True, distributor_id='dist-4')
dist_controller.add_distributor(
'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True,
distributor_id='dist-5')
dist_controller.add_distributor(
'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True,
distributor_id='dist-1')
dist_controller.add_distributor(
'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.distributor_manager = manager_factory.repo_distributor_manager()
with mock.patch("pulp.server.managers.repo.distributor.model.Repository.objects"):
# Populate the database with a repo with units
self.distributor_manager.add_distributor(
"repo-1", "mock-distributor", {"relative_url": "/a/bc/d"}, True, distributor_id="dist-1"
)
self.distributor_manager.add_distributor(
"repo-1", "mock-distributor", {"relative_url": "/a/c"}, True, distributor_id="dist-2"
)
self.distributor_manager.add_distributor(
"repo-2", "mock-distributor", {"relative_url": "/a/bc/e"}, True, distributor_id="dist-3"
)
self.distributor_manager.add_distributor("repo-3", "mock-distributor", {}, True, distributor_id="dist-4")
self.distributor_manager.add_distributor(
"repo-4", "mock-distributor", {"relative_url": "repo-5"}, True, distributor_id="dist-5"
)
self.distributor_manager.add_distributor(
"repo-5", "mock-distributor", {"relative_url": "a/bcd/e"}, True, distributor_id="dist-1"
)
self.distributor_manager.add_distributor(
"repo-6", "mock-distributor", {"relative_url": "a/bcde/f/"}, True, distributor_id="dist-1"
)
self.conduit = RepoConfigConduit("rpm")
def test_validate_config(self, mock_validate_config):
repo = Repository('test')
config = PluginCallConfiguration(None, None)
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
self.distributor.validate_config(repo, config, conduit)
mock_validate_config.assert_called_once_with(repo, config, conduit)
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
with mock.patch(
'pulp.server.controllers.distributor.model.Repository.objects'
):
# Populate the database with a repo with units
dist_controller.add_distributor('repo-1',
'mock-distributor',
{"relative_url": "/a/bc/d"},
True,
distributor_id='dist-1')
dist_controller.add_distributor('repo-1',
'mock-distributor',
{"relative_url": "/a/c"},
True,
distributor_id='dist-2')
dist_controller.add_distributor('repo-2',
'mock-distributor',
{"relative_url": "/a/bc/e"},
True,
distributor_id='dist-3')
dist_controller.add_distributor('repo-3',
'mock-distributor', {},
True,
distributor_id='dist-4')
dist_controller.add_distributor('repo-4',
'mock-distributor',
{"relative_url": "repo-5"},
True,
distributor_id='dist-5')
dist_controller.add_distributor('repo-5',
'mock-distributor',
{"relative_url": "a/bcd/e"},
True,
distributor_id='dist-1')
dist_controller.add_distributor('repo-6',
'mock-distributor',
{"relative_url": "a/bcde/f/"},
True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def update_distributor_config(repo_group_id, distributor_id,
distributor_config):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If
not, this method will raise an error and the existing configuration
will remain unchanged.
:param repo_group_id: identifies the group
:type repo_group_id: str
:param distributor_id: identifies the distributor on the group
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given group or distributor do not exist
:raise PulpDataException: if the plugin indicates the new configuration is invalid
"""
# Validation - calls will raise MissingResource
group_manager = manager_factory.repo_group_query_manager()
group = group_manager.get_group(repo_group_id)
distributor = RepoGroupDistributorManager.get_distributor(
repo_group_id, distributor_id)
distributor_type_id = distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id(
distributor_type_id)
# Resolve the requested changes into the existing config
merged_config = process_update_config(distributor['config'],
distributor_config)
# Request the distributor validate the new configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_group = common_utils.to_transfer_repo_group(group)
transfer_group.working_dir = common_utils.group_distributor_working_dir(
distributor_type_id, repo_group_id)
config_conduit = RepoConfigConduit(distributor_type_id)
# Request the plugin validate the configuration
try:
is_valid, message = distributor_instance.validate_config(
transfer_group, call_config, config_conduit)
if not is_valid:
raise PulpDataException(message)
except Exception, e:
msg = _(
'Exception received from distributor [%(d)s] while validating config'
)
msg = msg % {'d': distributor_type_id}
logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def test_validate_config_missing_required(self, mock_check):
repo = Repository('test')
config = self._generate_call_config(http=True, https=False)
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
valid, reasons = configuration.validate_config(repo, config, conduit)
self.assertFalse(valid)
expected_reason = 'Configuration key [relative_url] is required, but was not provided'
self.assertEqual(reasons, expected_reason)
mock_check.assert_called_once_with(repo, config.flatten(), conduit, [expected_reason])
def test_validate_config_unsupported_keys(self, mock_check):
repo = Repository('test')
config = self._generate_call_config(http=True, https=False, relative_url=None, foo='bar')
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
valid, reasons = configuration.validate_config(repo, config, conduit)
self.assertFalse(valid)
expected_reason = 'Configuration key [foo] is not supported'
self.assertEqual(reasons, expected_reason)
self.assertEqual(mock_check.call_count, 1)
def test_validate_config_https_http_false(self, mock_check):
repo = Repository('test')
config = self._generate_call_config(http=False, https=False, relative_url=None)
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
valid, reasons = configuration.validate_config(repo, config, conduit)
self.assertFalse(valid)
expected_reason = ('Settings serve via http and https are both set to false.'
' At least one option should be set to true.')
self.assertEqual(reasons, expected_reason)
self.assertEqual(mock_check.call_count, 1)
def test_validate_config__repocfg_gpg_cmd(self):
repo = Repository('test')
config = self._generate_call_config(http=False,
https=True,
relative_url="a/b")
config.repo_plugin_config["gpg_cmd"] = "this should fail"
conduit = RepoConfigConduit(TYPE_ID_DISTRIBUTOR_YUM)
valid, reasons = configuration.validate_config(repo, config, conduit)
self.assertFalse(valid)
expected_reason = ('Configuration key [gpg_cmd] is not allowed '
'in repository plugin configuration')
self.assertEqual(reasons, expected_reason)
def add_distributor(repo_group_id,
distributor_type_id,
group_plugin_config,
distributor_id=None):
"""
Adds an association from the given repository group to a distributor.
The assocation will be tracked through the distributor_id; each
distributor on a given group must have a unique ID. If this is not
specified, one will be generated. If a distributor already exists on the
group with a given ID, the existing one will be removed and replaced
with the newly configured one.
@param repo_group_id: identifies the repo group
@type repo_group_id: str
@param distributor_type_id: type of distributor being added; must reference
one of the installed group distributors
@type distributor_type_id: str
@param group_plugin_config: config to use for the distributor for this group alone
@type group_plugin_config: dict
@param distributor_id: if specified, the newly added distributor will be
referenced by this value and the group id; if omitted one will
be generated
@type distributor_id: str
@return: database representation of the added distributor
@rtype: dict
@raise MissingResource: if the group doesn't exist
@raise InvalidValue: if a distributor ID is provided and is not valid
@raise PulpDataException: if the plugin indicates the config is invalid
@raise PulpExecutionException: if the plugin raises an exception while
initializing the newly added distributor
"""
distributor_coll = RepoGroupDistributor.get_collection()
query_manager = manager_factory.repo_group_query_manager()
# Validation
group = query_manager.get_group(
repo_group_id) # will raise MissingResource
if not plugin_api.is_valid_group_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for the distributor on this repo
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id(
distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
clean_config = None
if group_plugin_config is not None:
clean_config = dict([(k, v)
for k, v in group_plugin_config.items()
if v is not None])
# Let the plugin validate the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_group = common_utils.to_transfer_repo_group(group)
config_conduit = RepoConfigConduit(distributor_type_id)
# Request the plugin validate the configuration
try:
is_valid, message = distributor_instance.validate_config(
transfer_group, call_config, config_conduit)
if not is_valid:
raise PulpDataException(message)
except Exception, e:
msg = _(
'Exception received from distributor [%(d)s] while validating config'
)
msg = msg % {'d': distributor_type_id}
_logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
class RepoConfigConduitTests(base.PulpServerTests):
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
with mock.patch('pulp.server.controllers.distributor.model.Repository.objects'):
# Populate the database with a repo with units
dist_controller.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True,
distributor_id='dist-1')
dist_controller.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True,
distributor_id='dist-2')
dist_controller.add_distributor(
'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True,
distributor_id='dist-3')
dist_controller.add_distributor(
'repo-3', 'mock-distributor', {}, True, distributor_id='dist-4')
dist_controller.add_distributor(
'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True,
distributor_id='dist-5')
dist_controller.add_distributor(
'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True,
distributor_id='dist-1')
dist_controller.add_distributor(
'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def tearDown(self):
super(RepoConfigConduitTests, self).tearDown()
mock_plugins.reset()
def clean(self):
super(RepoConfigConduitTests, self).clean()
mock_plugins.MOCK_DISTRIBUTOR.reset_mock()
model.Repository.objects.delete()
model.Distributor.objects.delete()
def test_get_distributors_by_relative_url_with_same_url(self):
"""
Test for distributors with an identical relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
def test_get_distributors_by_relative_url_with_with_excluded_repository_id(self):
"""
Test for distributors with a matching url but excluded because of the repo_id
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d", 'repo-1')
self.assertEqual(len(matches), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d", 'repo-1')
self.assertEqual(len(matches), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/", 'repo-1')
self.assertEqual(len(matches), 0)
def test_get_distributors_by_relative_url_with_different_url(self):
"""
Test for distributors with no matching relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/d")
self.assertEqual(len(matches), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("d")
self.assertEqual(len(matches), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("d/")
self.assertEqual(len(matches), 0)
def test_get_distributors_by_relative_url_with_superset_url(self):
"""
Test for distributors with urls that be overridden by the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc")
self.assertEqual(len(matches), 2)
# verify that the correct 2 repositories were found
matches = sorted(matches, key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc")
self.assertEqual(len(matches), 2)
# verify that the correct 2 repositories were found
matches = sorted(matches, key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/")
self.assertEqual(len(matches), 2)
# verify that the correct 2 repositories were found
matches = sorted(matches, key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
def test_get_distributors_by_relative_url_with_subset_url(self):
"""
Test for distributors with urls that would override the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e/")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-1')
def test_get_distributors_with_no_relative_url(self):
"""
Test for distributors where no relative url is specified, the distributor id is used
in it's place
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/repo-3")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-3')
matches = self.conduit.get_repo_distributors_by_relative_url("repo-3")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-3')
matches = self.conduit.get_repo_distributors_by_relative_url("repo-3/")
self.assertEqual(len(matches), 1)
self.assertEquals(matches[0]['repo_id'], 'repo-3')
def test_get_distributors_without_leading_or_trailing_slash_relative_url(self):
"""
Test matching repos that include preceding slashes with a search that doesn't include
preceding and trailing slashes.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bcd/e")
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0]['repo_id'], 'repo-5')
def test_get_distributors_without_leading_slash_relative_url(self):
"""
Test matching repos that do not include preceding slashes with a search that doesn't
include preceding and trailing slashes.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bcde/f")
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0]['repo_id'], 'repo-6')
def test_get_distributors_url_does_not_conflict_with_repo_id(self):
"""
Test matching repos with a defined relative-url using their repo id instead of relative-url.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("repo-1")
self.assertEqual(len(matches), 0)
def add_distributor(repo_id,
distributor_type_id,
repo_plugin_config,
auto_publish,
distributor_id=None):
"""
Adds an association from the given repository to a distributor. The
association will be tracked through the distributor_id; each distributor
on a given repository must have a unique ID. If this is not specified,
one will be generated. If a distributor already exists on the repo for
the given ID, the existing one will be removed and replaced with the
newly configured one.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_type_id: identifies the distributor; must correspond to a
distributor loaded at server startup
:type distributor_type_id: str
:param repo_plugin_config: configuration the repo will use with this
distributor; may be None
:type repo_plugin_config: dict
:param auto_publish: if true, this distributor will be invoked at the end
of every sync
:type auto_publish: bool
:param distributor_id: unique ID to refer to this distributor for this repo
:type distributor_id: str
:return: ID assigned to the distributor (only valid in
conjunction with the repo)
:raise MissingResource: if the given repo_id does not refer to a valid repo
:raise InvalidValue: if the distributor ID is provided and unacceptable
:raise InvalidDistributorConfiguration: if the distributor plugin does not accept the given
configuration
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repository=repo_id)
if not plugin_api.is_valid_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for this distributor on this repo; will be
# unique for all distributors on this repository but not globally
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(
distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items()
if v is not None])
else:
clean_config = None
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
distributor_type_id, repo_id)
config_conduit = RepoConfigConduit(distributor_type_id)
try:
result = distributor_instance.validate_config(
transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
logger.exception(
'Exception received from distributor [%s] while validating config'
% distributor_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def update_distributor_config(repo_id,
distributor_id,
distributor_config,
auto_publish=None):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_id: identifies the distributor on the repo
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:param auto_publish: If true, this distributor is used automatically during a sync operation
:type auto_publish: bool
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given repo or distributor doesn't exist
:raise PulpDataException: if the plugin rejects the given changes
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Input Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repository=repo_id)
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
if repo_distributor is None:
raise MissingResource(distributor=distributor_id)
distributor_type_id = repo_distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(
distributor_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_distributor['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [
k for k in distributor_config if distributor_config[k] is None
]
for key in unset_property_names:
merged_config.pop(key, None)
distributor_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(distributor_config)
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
distributor_type_id, repo_id)
config_conduit = RepoConfigConduit(distributor_type_id)
try:
result = distributor_instance.validate_config(
transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
msg = _(
'Exception raised from distributor [%(d)s] while validating config for repo '
'[%(r)s]')
msg = msg % {'d': distributor_type_id, 'r': repo_id}
logger.exception(msg)
raise PulpDataException(e.args), None, sys.exc_info()[2]
class RepoConfigConduitTests(base.PulpServerTests):
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"},
True, distributor_id='dist-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/c"},
True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor('repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"},
True, distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor('repo-4', 'mock-distributor', {"relative_url": "/repo-5"},
True, distributor_id='dist-5')
self.conduit = RepoConfigConduit('rpm')
def clean(self):
super(RepoConfigConduitTests, self).clean()
mock_plugins.MOCK_DISTRIBUTOR.reset_mock()
Repo.get_collection().remove()
RepoDistributor.get_collection().remove()
def test_get_distributors_by_relative_url_with_same_url(self):
"""
Test for distributors with an identical relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
def test_get_distributors_by_relative_url_with_with_excluded_repository_id(self):
"""
Test for distributors with a matching url but excluded because of the repo_id
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d", 'repo-1')
self.assertEquals(matches.count(), 0)
def test_get_distributors_by_relative_url_with_different_url(self):
"""
Test for distributors with no matching relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/d")
self.assertEquals(matches.count(), 0)
def test_get_distributors_by_relative_url_with_superset_url(self):
"""
Test for distributors with urls that be overridden by the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc")
self.assertEquals(matches.count(), 2)
#verify that the correct 2 repositories were found
matches = sorted(list(matches), key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
def test_get_distributors_by_relative_url_with_subset_url(self):
"""
Test for distributors with urls that would override the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d/e")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
def test_get_distributors_with_no_relative_url(self):
"""
Test for distributors where no relative url is specified, the distributor id is used
in it's place
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/repo-3")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-3')
def update_distributor_config(repo_id, distributor_id, distributor_config, auto_publish=None):
"""
Attempts to update the saved configuration for the given distributor.
The distributor will be asked if the new configuration is valid. If not,
this method will raise an error and the existing configuration will
remain unchanged.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_id: identifies the distributor on the repo
:type distributor_id: str
:param distributor_config: new configuration values to use
:type distributor_config: dict
:param auto_publish: If true, this distributor is used automatically during a sync operation
:type auto_publish: bool
:return: the updated distributor
:rtype: dict
:raise MissingResource: if the given repo or distributor doesn't exist
:raise PulpDataException: if the plugin rejects the given changes
"""
distributor_coll = RepoDistributor.get_collection()
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
repo_distributor = distributor_coll.find_one({'repo_id': repo_id, 'id': distributor_id})
if repo_distributor is None:
raise MissingResource(distributor=distributor_id)
distributor_type_id = repo_distributor['distributor_type_id']
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id)
# The supplied config is a delta of changes to make to the existing config.
# The plugin expects a full configuration, so we apply those changes to
# the original config and pass that to the plugin's validate method.
merged_config = dict(repo_distributor['config'])
# The convention is that None in an update is removing the value and
# setting it to the default. Find all such properties in this delta and
# remove them from the existing config if they are there.
unset_property_names = [k for k in distributor_config if distributor_config[k] is None]
for key in unset_property_names:
merged_config.pop(key, None)
distributor_config.pop(key, None)
# Whatever is left over are the changed/added values, so merge them in.
merged_config.update(distributor_config)
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, merged_config)
transfer_repo = repo_obj.to_transfer_repo()
config_conduit = RepoConfigConduit(distributor_type_id)
result = distributor_instance.validate_config(transfer_repo, call_config,
config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpDataException(message)
# Confirm that the auto_publish value is sane before updating the value, if it exists
if auto_publish is not None:
if isinstance(auto_publish, bool):
repo_distributor['auto_publish'] = auto_publish
else:
raise InvalidValue(['auto_publish'])
# If we got this far, the new config is valid, so update the database
repo_distributor['config'] = merged_config
distributor_coll.save(repo_distributor, safe=True)
return repo_distributor
def add_distributor(repo_id, distributor_type_id, repo_plugin_config,
auto_publish, distributor_id=None):
"""
Adds an association from the given repository to a distributor. The
association will be tracked through the distributor_id; each distributor
on a given repository must have a unique ID. If this is not specified,
one will be generated. If a distributor already exists on the repo for
the given ID, the existing one will be removed and replaced with the
newly configured one.
:param repo_id: identifies the repo
:type repo_id: str
:param distributor_type_id: identifies the distributor; must correspond to a
distributor loaded at server startup
:type distributor_type_id: str
:param repo_plugin_config: configuration the repo will use with this
distributor; may be None
:type repo_plugin_config: dict
:param auto_publish: if true, this distributor will be invoked at the end
of every sync
:type auto_publish: bool
:param distributor_id: unique ID to refer to this distributor for this repo
:type distributor_id: str
:return: ID assigned to the distributor (only valid in
conjunction with the repo)
:raise MissingResource: if the given repo_id does not refer to a valid repo
:raise InvalidValue: if the distributor ID is provided and unacceptable
:raise InvalidDistributorConfiguration: if the distributor plugin does not accept the given
configuration
"""
distributor_coll = RepoDistributor.get_collection()
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_distributor(distributor_type_id):
raise InvalidValue(['distributor_type_id'])
# Determine the ID for this distributor on this repo; will be
# unique for all distributors on this repository but not globally
if distributor_id is None:
distributor_id = str(uuid.uuid4())
else:
# Validate if one was passed in
if not is_distributor_id_valid(distributor_id):
raise InvalidValue(['distributor_id'])
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
config_conduit = RepoConfigConduit(distributor_type_id)
transfer_repo = repo_obj.to_transfer_repo()
result = distributor_instance.validate_config(transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpDataException(message)
# Remove the old distributor if it exists
try:
RepoDistributorManager.remove_distributor(repo_id, distributor_id)
except MissingResource:
pass # if it didn't exist, no problem
# Let the distributor plugin initialize the repository
try:
distributor_instance.distributor_added(transfer_repo, call_config)
except Exception:
msg = _('Error initializing distributor [%(d)s] for repo [%(r)s]')
msg = msg % {'d': distributor_type_id, 'r': repo_id}
_logger.exception(msg)
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
distributor = RepoDistributor(repo_id, distributor_id, distributor_type_id, clean_config,
auto_publish)
distributor_coll.save(distributor, safe=True)
return distributor
class RepoConfigConduitTests(base.PulpServerTests):
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True,
distributor_id='dist-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor(
'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True,
distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor(
'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True, distributor_id='dist-5')
self.repo_manager.create_repo('repo-5')
self.distributor_manager.add_distributor(
'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True,
distributor_id='dist-1')
self.repo_manager.create_repo('repo-6')
self.distributor_manager.add_distributor(
'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def tearDown(self):
super(RepoConfigConduitTests, self).tearDown()
mock_plugins.reset()
def clean(self):
super(RepoConfigConduitTests, self).clean()
mock_plugins.MOCK_DISTRIBUTOR.reset_mock()
Repo.get_collection().remove()
RepoDistributor.get_collection().remove()
def test_get_distributors_by_relative_url_with_same_url(self):
"""
Test for distributors with an identical relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
def test_get_distributors_by_relative_url_with_with_excluded_repository_id(self):
"""
Test for distributors with a matching url but excluded because of the repo_id
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc/d", 'repo-1')
self.assertEquals(matches.count(), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d", 'repo-1')
self.assertEquals(matches.count(), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/", 'repo-1')
self.assertEquals(matches.count(), 0)
def test_get_distributors_by_relative_url_with_different_url(self):
"""
Test for distributors with no matching relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/d")
self.assertEquals(matches.count(), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("d")
self.assertEquals(matches.count(), 0)
matches = self.conduit.get_repo_distributors_by_relative_url("d/")
self.assertEquals(matches.count(), 0)
def test_get_distributors_by_relative_url_with_superset_url(self):
"""
Test for distributors with urls that be overridden by the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/a/bc")
self.assertEquals(matches.count(), 2)
# verify that the correct 2 repositories were found
matches = sorted(list(matches), key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc")
self.assertEquals(matches.count(), 2)
# verify that the correct 2 repositories were found
matches = sorted(list(matches), key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/")
self.assertEquals(matches.count(), 2)
# verify that the correct 2 repositories were found
matches = sorted(list(matches), key=itemgetter('repo_id'))
self.assertEquals(matches[0]['repo_id'], 'repo-1')
self.assertEquals(matches[1]['repo_id'], 'repo-2')
def test_get_distributors_by_relative_url_with_subset_url(self):
"""
Test for distributors with urls that would override the proposed relative url
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
matches = self.conduit.get_repo_distributors_by_relative_url("a/bc/d/e/")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-1')
def test_get_distributors_with_no_relative_url(self):
"""
Test for distributors where no relative url is specified, the distributor id is used
in it's place
"""
matches = self.conduit.get_repo_distributors_by_relative_url("/repo-3")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-3')
matches = self.conduit.get_repo_distributors_by_relative_url("repo-3")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-3')
matches = self.conduit.get_repo_distributors_by_relative_url("repo-3/")
self.assertEquals(matches.count(), 1)
self.assertEquals(next(matches)['repo_id'], 'repo-3')
def test_get_distributors_without_leading_or_trailing_slash_relative_url(self):
"""
Test matching repos that include preceding slashes with a search that doesn't include
preceding and trailing slashes.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bcd/e")
self.assertEqual(matches.count(), 1)
self.assertEqual(next(matches)['repo_id'], 'repo-5')
def test_get_distributors_without_leading_slash_relative_url(self):
"""
Test matching repos that do not include preceding slashes with a search that doesn't
include preceding and trailing slashes.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("a/bcde/f")
self.assertEqual(matches.count(), 1)
self.assertEqual(next(matches)['repo_id'], 'repo-6')
def test_get_distributors_url_does_not_conflict_with_repo_id(self):
"""
Test matching repos with a defined relative-url using their repo id instead of relative-url.
"""
matches = self.conduit.get_repo_distributors_by_relative_url("repo-1")
self.assertEqual(matches.count(), 0)
def add_distributor(repo_id, distributor_type_id, repo_plugin_config,
auto_publish, distributor_id=None):
"""
Adds an association from the given repository to a distributor. The distributor_id is unique for
a given repository. If distributor_id is not specified, one will be generated. If a distributor
already exists on the repo for the given ID, the existing one will be removed and replaced with
the newly configured one.
:param repo_id: identifies the repo
:type repo_id: basestring
:param distributor_type_id: must correspond to a distributor type loaded at server startup
:type distributor_type_id: basestring
:param repo_plugin_config: configuration the repo will use with this distributor
:type repo_plugin_config: dict or None
:param auto_publish: if True, this distributor will be invoked at the end of every sync
:type auto_publish: bool
:param distributor_id: unique ID to refer to this distributor for this repo
:type distributor_id: basestring
:return: distributor object
:rtype: pulp.server.db.model.Distributor
:raise InvalidValue: if the distributor ID is provided and unacceptable
:raise exceptions.PulpDataException: if the plugin returns that the config is invalid
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_distributor(distributor_type_id):
raise exceptions.InvalidValue(['distributor_type_id'])
if distributor_id is None:
distributor_id = str(uuid.uuid4())
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(distributor_type_id)
# Remove any keys whose values are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
config_conduit = RepoConfigConduit(distributor_type_id)
transfer_repo = repo_obj.to_transfer_repo()
result = distributor_instance.validate_config(transfer_repo, call_config, config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpDataException(message)
try:
model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=distributor_id)
delete(repo_id, distributor_id)
except exceptions.MissingResource:
pass # if it didn't exist, no problem
distributor_instance.distributor_added(transfer_repo, call_config)
distributor = model.Distributor(repo_id, distributor_id, distributor_type_id, clean_config,
auto_publish)
distributor.save()
return distributor
def update(repo_id, dist_id, config=None, delta=None):
"""
Update the distributor and (re)bind any bound consumers.
:param distributor: distributor to be updated
:type distributor: pulp.server.db.model.Distributor
:param config: A configuration dictionary for a distributor instance. The contents of this dict
depends on the type of distributor. Values of None will remove they key from the
config. Keys ommited from this dictionary will remain unchanged.
:type config: dict
:param delta: A dictionary used to change conf values for a distributor instance. This currently
only supports the 'auto_publish' keyword, which should have a value of type bool
:type delta: dict or None
:return: result containing any errors and tasks spawned
:rtype pulp.server.async.tasks.TaskResult
"""
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
distributor = model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=dist_id)
for k, v in config.iteritems():
if v is None:
distributor.config.pop(k)
else:
distributor.config[k] = v
auto_publish = delta.get('auto_publish') if delta else None
if isinstance(auto_publish, bool):
distributor.auto_publish = auto_publish
elif not isinstance(auto_publish, type(None)):
raise exceptions.InvalidValue(['auto_publish'])
# Let the distributor plugin verify the configuration
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(
distributor.distributor_type_id)
call_config = PluginCallConfiguration(plugin_config, distributor.config)
transfer_repo = repo.to_transfer_repo()
config_conduit = RepoConfigConduit(distributor.distributor_type_id)
result = distributor_instance.validate_config(transfer_repo, call_config,
config_conduit)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpDataException(message)
distributor.save()
unbind_errors = []
additional_tasks = []
options = {}
bind_manager = managers.consumer_bind_manager()
for bind in bind_manager.find_by_distributor(distributor.repo_id, distributor.distributor_id):
try:
report = bind_manager.bind(bind['consumer_id'], bind['repo_id'], bind['distributor_id'],
bind['notify_agent'], bind['binding_config'], options)
if report:
additional_tasks.extend(report.spawned_tasks)
except Exception, e:
unbind_errors.append(e)
