def test_process_main_missing_slashes(self, mock_deb_download, mock_deb_packagefile): """ Test when the the feed is missing a '/' at the end and the package_file_path is not relative. """ mock_deb_packagefile.return_value = [ { 'Package': 'foo', 'Version': '1.5', 'Architecture': 'x86_64', 'Size': '105', 'Filename': 'foo.deb' } ] plugin_config = { importer_constants.KEY_FEED: 'http://ftp.fau.de/debian', 'package-file-path': '/dists/stable/main/binary-amd64/' } self.config = PluginCallConfiguration({}, plugin_config) self.step.config = plugin_config self.step.parent.available_units = [] self.step.process_main() download_feed = 'http://ftp.fau.de/debian/dists/stable/main/binary-amd64/Packages' download_location = os.path.join(self.working_dir, 'Packages') mock_deb_download.assert_called_once_with(download_feed, download_location) self.assertEquals(len(self.step.parent.available_units), 1) self.assertDictEqual(self.step.parent.available_units[0], {'name': 'foo', 'version': '1.5', 'architecture': 'x86_64'})
class PluginCallConfigurationTests(unittest.TestCase): def setUp(self): super(PluginCallConfigurationTests, self).setUp() self.override_config = {'a': 'a4', 'b': 'b4', 'e': 'e4'} self.repo_plugin_config = {'a': 'a3', 'c': 'c3'} self.plugin_config = {'a': 'a2', 'b': 'b2', 'd': 'd2'} self.default_config = {'a': 'a1'} self.config = PluginCallConfiguration(self.plugin_config, self.repo_plugin_config, self.override_config) self.config.default_config = self.default_config def test_flatten(self): # Test flattened = self.config.flatten() # Verify self.assertTrue(isinstance(flattened, dict)) self.assertEqual(5, len(flattened)) self.assertEqual(flattened['a'], 'a4') self.assertEqual(flattened['b'], 'b4') self.assertEqual(flattened['c'], 'c3') self.assertEqual(flattened['d'], 'd2') self.assertEqual(flattened['e'], 'e4')
def test_import_unit_files_already_exist_size_mismatch(self, *mocks): # Setup self.populate() pulp_conf.set('server', 'storage_dir', self.parentfs) dist = NodesHttpDistributor() working_dir = os.path.join(self.childfs, 'working_dir') os.makedirs(working_dir) repo = Repository(self.REPO_ID, working_dir) cfg = self.dist_conf() conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) dist.publish_repo(repo, conduit, cfg) Repo.get_collection().remove() RepoDistributor.get_collection().remove() RepoContentUnit.get_collection().remove() unit_db.clean() self.define_plugins() parent_content = os.path.join(self.parentfs, 'content') child_content = os.path.join(self.childfs, 'content') shutil.copytree(parent_content, child_content) for fn in os.listdir(child_content): path = os.path.join(child_content, fn) if os.path.isdir(path): continue with open(path, 'w') as fp: fp.truncate() # Test importer = NodesHttpImporter() publisher = dist.publisher(repo, cfg) manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) configuration = { constants.MANIFEST_URL_KEYWORD: manifest_url, constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, } configuration = PluginCallConfiguration(configuration, {}) conduit = RepoSyncConduit( self.REPO_ID, constants.HTTP_IMPORTER, RepoContentUnit.OWNER_TYPE_IMPORTER, constants.HTTP_IMPORTER) pulp_conf.set('server', 'storage_dir', self.childfs) importer.sync_repo(repo, conduit, configuration) # Verify units = conduit.get_units() self.assertEquals(len(units), self.NUM_UNITS) mock_importer_config_to_nectar_config = mocks[0] mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())
def test_import(self, *mocks): # Setup self.populate() max_concurrency = 5 max_bandwidth = 12345 pulp_conf.set('server', 'storage_dir', self.parentfs) dist = NodesHttpDistributor() working_dir = os.path.join(self.childfs, 'working_dir') os.makedirs(working_dir) repo = Repository(self.REPO_ID, working_dir) cfg = self.dist_conf() conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) dist.publish_repo(repo, conduit, cfg) Repo.get_collection().remove() RepoDistributor.get_collection().remove() RepoContentUnit.get_collection().remove() unit_db.clean() self.define_plugins() # Test importer = NodesHttpImporter() publisher = dist.publisher(repo, cfg) manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) configuration = { constants.MANIFEST_URL_KEYWORD: manifest_url, constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, importer_constants.KEY_MAX_DOWNLOADS: max_concurrency, importer_constants.KEY_MAX_SPEED: max_bandwidth, } configuration = PluginCallConfiguration(configuration, {}) conduit = RepoSyncConduit( self.REPO_ID, constants.HTTP_IMPORTER, RepoContentUnit.OWNER_TYPE_IMPORTER, constants.HTTP_IMPORTER) pulp_conf.set('server', 'storage_dir', self.childfs) importer.sync_repo(repo, conduit, configuration) # Verify units = conduit.get_units() self.assertEquals(len(units), self.NUM_UNITS) mock_importer_config_to_nectar_config = mocks[0] mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())
def setUp(self): super(PluginCallConfigurationTests, self).setUp() self.override_config = {'a': 'a4', 'b': 'b4', 'e': 'e4'} self.repo_plugin_config = {'a': 'a3', 'c': 'c3'} self.plugin_config = {'a': 'a2', 'b': 'b2', 'd': 'd2'} self.default_config = {'a': 'a1'} self.config = PluginCallConfiguration(self.plugin_config, self.repo_plugin_config, self.override_config) self.config.default_config = self.default_config
def setUp(self): self.working_dir = tempfile.mkdtemp() self.repo = RepositoryModel('repo1') self.repo.working_dir = self.working_dir self.conduit = mock.MagicMock() plugin_config = { importer_constants.KEY_FEED: 'http://ftp.fau.de/debian/dists/stable/main/binary-amd64/', } self.config = PluginCallConfiguration({}, plugin_config) self.step = sync.GetMetadataStep(repo=self.repo, conduit=self.conduit, config=self.config, working_dir=self.working_dir) self.step.parent = mock.MagicMock() self.index = self.step.parent.index_repository
def test_import_unit_files_already_exist(self, mock_get_working, *mocks): # Setup self.populate() mock_get_working.return_value = self.temp_dir with mock_config.patch({'server': {'storage_dir': self.parentfs}}): dist = NodesHttpDistributor() working_dir = os.path.join(self.childfs, 'working_dir') os.makedirs(working_dir) repo = Repository(self.REPO_ID, working_dir) cfg = self.dist_conf() conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) dist.publish_repo(repo, conduit, cfg) model.Distributor.objects.delete() RepoContentUnit.get_collection().remove() unit_db.clean() self.define_plugins() parent_content = os.path.join(self.parentfs, 'content') child_content = os.path.join(self.childfs, 'content') shutil.copytree(parent_content, child_content) # Test importer = NodesHttpImporter() publisher = dist.publisher(repo, cfg) manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) configuration = { constants.MANIFEST_URL_KEYWORD: manifest_url, constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, } configuration = PluginCallConfiguration(configuration, {}) conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) with mock_config.patch({'server': {'storage_dir': self.childfs}}): with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): importer.sync_repo(repo, conduit, configuration) # Verify units = conduit.get_units() self.assertEquals(len(units), self.NUM_UNITS) mock_importer_config_to_nectar_config = mocks[0] mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())
def test_process_step_skip_units(self): self.publisher.config = PluginCallConfiguration(None, {'skip': ['FOO']}) step = publish_step.UnitPublishStep('foo_step', 'FOO') step.parent = self.publisher step.process() self.assertEquals(step.state, reporting_constants.STATE_SKIPPED)
def add_distributor(self, repo_id, distributor_type_id, repo_plugin_config, auto_publish, distributor_id=None): """ Adds an association from the given repository to a distributor. The association will be tracked through the distributor_id; each distributor on a given repository must have a unique ID. If this is not specified, one will be generated. If a distributor already exists on the repo for the given ID, the existing one will be removed and replaced with the newly configured one. @param repo_id: identifies the repo @type repo_id: str @param distributor_type_id: identifies the distributor; must correspond to a distributor loaded at server startup @type distributor_type_id: str @param repo_plugin_config: configuration the repo will use with this distributor; may be None @type repo_plugin_config: dict @param auto_publish: if true, this distributor will be invoked at the end of every sync @type auto_publish: bool @param distributor_id: unique ID to refer to this distributor for this repo @type distributor_id: str @return: ID assigned to the distributor (only valid in conjunction with the repo) @raise MissingResource: if the given repo_id does not refer to a valid repo @raise InvalidValue: if the distributor ID is provided and unacceptable @raise InvalidDistributorConfiguration: if the distributor plugin does not accept the given configuration """ repo_coll = Repo.get_collection() distributor_coll = RepoDistributor.get_collection() # Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repository=repo_id) if not plugin_api.is_valid_distributor(distributor_type_id): raise InvalidValue(['distributor_type_id']) # Determine the ID for this distributor on this repo; will be # unique for all distributors on this repository but not globally if distributor_id is None: distributor_id = str(uuid.uuid4()) else: # Validate if one was passed in if not is_distributor_id_valid(distributor_id): raise InvalidValue(['distributor_id']) distributor_instance, plugin_config = plugin_api.get_distributor_by_id( distributor_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the distributor plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.distributor_working_dir( distributor_type_id, repo_id) query_manager = manager_factory.repo_query_manager() related_repos = query_manager.find_with_distributor_type( distributor_type_id) transfer_related_repos = [] for r in related_repos: all_configs = [d['config'] for d in r['distributors']] trr = common_utils.to_related_repo(r, all_configs) transfer_related_repos.append(trr) try: result = distributor_instance.validate_config( transfer_repo, call_config, transfer_related_repos) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: _LOG.exception( 'Exception received from distributor [%s] while validating config' % distributor_type_id) raise PulpDataException(e.args), None, sys.exc_info()[2]
def test_no_dirs(self): config = PluginCallConfiguration({}, {}) repo = mock.Mock(id='foo') dirs = configuration.get_export_repo_group_publish_dirs(repo, config) self.assertEquals(dirs, [])
def test_is_skipped_dict(self): step = publish_step.UnitPublishStep("foo", 'bar') step.config = PluginCallConfiguration(None, {'skip': {'bar': True, 'baz': True}}) self.assertTrue(step.is_skipped())
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config.validate_max_speed(config)
def import_uploaded_unit(repo_id, unit_type_id, unit_key, unit_metadata, upload_id, override_config=None): """ Called to trigger the importer's handling of an uploaded unit. This should not be called until the bits have finished uploading. The importer is then responsible for moving the file to the correct location, adding it to the Pulp server's inventory, and associating it with the repository. This call will first call is_valid_upload to check the integrity of the destination repository. See that method's documentation for exception possibilities. :param repo_id: identifies the repository into which the unit is uploaded :type repo_id: str :param unit_type_id: type of unit being uploaded :type unit_type_id: str :param unit_key: unique identifier for the unit (user-specified) :type unit_key: dict :param unit_metadata: any user-specified information about the unit :type unit_metadata: dict :param upload_id: upload being imported :type upload_id: str :return: A SyncReport indicating the success or failure of the upload :rtype: pulp.plugins.model.SyncReport """ # If it doesn't raise an exception, it's good to go ContentUploadManager.is_valid_upload(repo_id, unit_type_id) repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Assemble the data needed for the import conduit = UploadConduit(repo_id, repo_importer['id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], override_config) transfer_repo = repo_obj.to_transfer_repo() file_path = ContentUploadManager._upload_file_path(upload_id) # Invoke the importer try: result = importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key, unit_metadata, file_path, conduit, call_config) repo_controller.rebuild_content_unit_counts(repo_obj) return result except PulpException: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise except Exception, e: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise PulpExecutionException(e), None, sys.exc_info()[2]
def test_valid(self): config = PluginCallConfiguration( {}, {importer_constants.KEY_PROXY_PASS: '******', importer_constants.KEY_PROXY_USER: '******'}) importer_config.validate_proxy_password(config)
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config.validate_feed_requirement(config)
def test_valid_str(self): config = PluginCallConfiguration({}, {importer_constants.KEY_PROXY_PORT: '3128'}) importer_config.validate_proxy_port(config)
def test_non_string(self): config = PluginCallConfiguration({}, {importer_constants.KEY_PROXY_PORT: 1.1}) try: importer_config.validate_proxy_port(config) except ValueError, e: self.assertTrue('1.1' in e[0])
def test_failure(self): config = PluginCallConfiguration({}, {importer_constants.KEY_VALIDATE: 0}) self.assertRaises(importer_config.InvalidConfig, importer_config.validate_config, config)
def test_validate(self): config = PluginCallConfiguration({}, {importer_constants.KEY_PROXY_HOST: 'http://fake.com/'}) importer_config.validate_proxy_host(config)
def test_all_pass(self): # All options are optional, so we can run them without concern config = PluginCallConfiguration({}, {}) importer_config.validate_config(config)
def test_validate_config_no_files_dir_specified(self): config = PluginCallConfiguration({}, {}) return_val, error_message = self.distributor.validate_config( self.repo, config, None) self.assertTrue(return_val) self.assertEquals(error_message, None)
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config.validate_proxy_password(config)
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config._run_validate_is_non_required_bool(config, 'missing')
def test_valid(self): config = PluginCallConfiguration({}, {importer_constants.KEY_SSL_CLIENT_CERT: 'cert'}) importer_config.validate_ssl_client_cert(config)
def test_str(self): config = PluginCallConfiguration({}, {importer_constants.KEY_MAX_SPEED: '512'}) importer_config.validate_max_speed(config)
def test_valid(self): config = PluginCallConfiguration({}, {importer_constants.KEY_SSL_CLIENT_KEY: 'key'}) importer_config.validate_ssl_client_key(config)
def test_validate(self): config = PluginCallConfiguration({}, {importer_constants.KEY_MAX_DOWNLOADS: 11}) importer_config.validate_max_downloads(config)
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config.validate_ssl_client_key(config)
def _generate_call_config(**kwargs): config = PluginCallConfiguration(None, None) config.default_config.update(kwargs) return config
def regenerate_applicability(profile_hash, content_type, profile_id, bound_repo_id, existing_applicability=None): """ Regenerate and save applicability data for given profile and bound repo id. If existing_applicability is not None, replace it with the new applicability data. :param profile_hash: hash of the unit profile :type profile_hash: basestring :param content_type: profile (unit) type ID :type content_type: str :param profile_id: unique id of the unit profile :type profile_id: str :param bound_repo_id: repo id to be used to calculate applicability against the given unit profile :type bound_repo_id: str :param existing_applicability: existing RepoProfileApplicability object to be replaced :type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability """ profiler_conduit = ProfilerConduit() # Get the profiler for content_type of given unit_profile profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler( content_type) # Check if the profiler supports applicability, else return if profiler.calculate_applicable_units == Profiler.calculate_applicable_units: # If base class calculate_applicable_units method is called, # skip applicability regeneration return # Find out which content types have unit counts greater than zero in the bound repo repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types( bound_repo_id) # Get the intersection of existing types in the repo and the types that the profiler # handles. If the intersection is not empty, regenerate applicability if (set(repo_content_types) & set(profiler.metadata()['types'])): # Get the actual profile for existing_applicability or lookup using profile_id if existing_applicability: profile = existing_applicability.profile else: unit_profile = UnitProfile.get_collection().find_one( {'id': profile_id}, projection=['profile']) profile = unit_profile['profile'] call_config = PluginCallConfiguration(plugin_config=profiler_cfg, repo_plugin_config=None) try: applicability = profiler.calculate_applicable_units( profile, bound_repo_id, call_config, profiler_conduit) except NotImplementedError: msg = "Profiler for content type [%s] does not support applicability" % content_type _logger.debug(msg) return try: # Create a new RepoProfileApplicability object and save it in the db RepoProfileApplicability.objects.create( profile_hash, bound_repo_id, profile, applicability) except DuplicateKeyError: # Update existing applicability if not existing_applicability: applicability_dict = RepoProfileApplicability.get_collection( ).find_one({ 'repo_id': bound_repo_id, 'profile_hash': profile_hash }) existing_applicability = RepoProfileApplicability( **applicability_dict) existing_applicability.applicability = applicability existing_applicability.save()
def test_is_skipped_dict_not_skipped(self): step = publish_step.UnitPublishStep("foo", 'bar') step.config = PluginCallConfiguration(None, None) self.assertFalse(step.is_skipped())
def test_str(self): config = PluginCallConfiguration({}, {importer_constants.KEY_UNITS_RETAIN_OLD_COUNT: '1'}) importer_config.validate_retain_old_count(config)
class TestGenerateMetadataStep(unittest.TestCase): def setUp(self): self.working_dir = tempfile.mkdtemp() self.repo = RepositoryModel('repo1') self.repo.working_dir = self.working_dir self.conduit = mock.MagicMock() plugin_config = { importer_constants.KEY_FEED: 'http://ftp.fau.de/debian/dists/stable/main/binary-amd64/', } self.config = PluginCallConfiguration({}, plugin_config) self.step = sync.GetMetadataStep(repo=self.repo, conduit=self.conduit, config=self.config, working_dir=self.working_dir) self.step.parent = mock.MagicMock() self.index = self.step.parent.index_repository def tearDown(self): super(TestGenerateMetadataStep, self).tearDown() shutil.rmtree(self.working_dir) @mock.patch('pulp_deb.plugins.importers.sync.debian_support.PackageFile') @mock.patch('pulp_deb.plugins.importers.sync.debian_support.download_file') def test_process_main(self, mock_deb_download, mock_deb_packagefile): mock_deb_packagefile.return_value = [ { 'Package': 'foo', 'Version': '1.5', 'Architecture': 'x86_64', 'Size': '105', 'Filename': 'foo.deb' } ] self.step.parent.available_units = [] self.step.process_main() download_feed = self.config.get(importer_constants.KEY_FEED) + 'Packages' download_location = os.path.join(self.working_dir, 'Packages') mock_deb_download.assert_called_once_with(download_feed, download_location) self.assertEquals(len(self.step.parent.available_units), 1) self.assertDictEqual(self.step.parent.available_units[0], {'name': 'foo', 'version': '1.5', 'architecture': 'x86_64'}) @mock.patch('pulp_deb.plugins.importers.sync.debian_support.PackageFile') @mock.patch('pulp_deb.plugins.importers.sync.debian_support.download_file') def test_process_main_sub_packagefile(self, mock_deb_download, mock_deb_packagefile): """ Test when the Packages file is not in the feed directory. """ mock_deb_packagefile.return_value = [ { 'Package': 'foo', 'Version': '1.5', 'Architecture': 'x86_64', 'Size': '105', 'Filename': 'foo.deb' } ] plugin_config = { importer_constants.KEY_FEED: 'http://ftp.fau.de/debian/', 'package-file-path': 'dists/stable/main/binary-amd64/' } self.config = PluginCallConfiguration({}, plugin_config) self.step.config = plugin_config self.step.parent.available_units = [] self.step.process_main() download_feed = 'http://ftp.fau.de/debian/dists/stable/main/binary-amd64/Packages' download_location = os.path.join(self.working_dir, 'Packages') mock_deb_download.assert_called_once_with(download_feed, download_location) self.assertEquals(len(self.step.parent.available_units), 1) self.assertDictEqual(self.step.parent.available_units[0], {'name': 'foo', 'version': '1.5', 'architecture': 'x86_64'}) @mock.patch('pulp_deb.plugins.importers.sync.debian_support.PackageFile') @mock.patch('pulp_deb.plugins.importers.sync.debian_support.download_file') def test_process_main_missing_slashes(self, mock_deb_download, mock_deb_packagefile): """ Test when the the feed is missing a '/' at the end and the package_file_path is not relative. """ mock_deb_packagefile.return_value = [ { 'Package': 'foo', 'Version': '1.5', 'Architecture': 'x86_64', 'Size': '105', 'Filename': 'foo.deb' } ] plugin_config = { importer_constants.KEY_FEED: 'http://ftp.fau.de/debian', 'package-file-path': '/dists/stable/main/binary-amd64/' } self.config = PluginCallConfiguration({}, plugin_config) self.step.config = plugin_config self.step.parent.available_units = [] self.step.process_main() download_feed = 'http://ftp.fau.de/debian/dists/stable/main/binary-amd64/Packages' download_location = os.path.join(self.working_dir, 'Packages') mock_deb_download.assert_called_once_with(download_feed, download_location) self.assertEquals(len(self.step.parent.available_units), 1) self.assertDictEqual(self.step.parent.available_units[0], {'name': 'foo', 'version': '1.5', 'architecture': 'x86_64'})
def test_valid(self): config = PluginCallConfiguration({}, {importer_constants.KEY_FEED: "http://test.com/feed"}) importer_config.validate_feed_requirement(config)
def update_distributor_config(self, repo_id, distributor_id, distributor_config): """ Attempts to update the saved configuration for the given distributor. The distributor will be asked if the new configuration is valid. If not, this method will raise an error and the existing configuration will remain unchanged. @param repo_id: identifies the repo @type repo_id: str @param distributor_id: identifies the distributor on the repo @type distributor_id: str @param distributor_config: new configuration values to use @type distributor_config: dict @return: the updated distributor @rtype: dict @raise MissingResource: if the given repo or distributor doesn't exist @raise PulpDataException: if the plugin rejects the given changes """ repo_coll = Repo.get_collection() distributor_coll = RepoDistributor.get_collection() # Input Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repository=repo_id) repo_distributor = distributor_coll.find_one({ 'repo_id': repo_id, 'id': distributor_id }) if repo_distributor is None: raise MissingResource(distributor=distributor_id) distributor_type_id = repo_distributor['distributor_type_id'] distributor_instance, plugin_config = plugin_api.get_distributor_by_id( distributor_type_id) # The supplied config is a delta of changes to make to the existing config. # The plugin expects a full configuration, so we apply those changes to # the original config and pass that to the plugin's validate method. merged_config = dict(repo_distributor['config']) # The convention is that None in an update is removing the value and # setting it to the default. Find all such properties in this delta and # remove them from the existing config if they are there. unset_property_names = [ k for k in distributor_config if distributor_config[k] is None ] for key in unset_property_names: merged_config.pop(key, None) distributor_config.pop(key, None) # Whatever is left over are the changed/added values, so merge them in. merged_config.update(distributor_config) # Let the distributor plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, merged_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.distributor_working_dir( distributor_type_id, repo_id) query_manager = manager_factory.repo_query_manager() related_repos = query_manager.find_with_distributor_type( distributor_type_id) transfer_related_repos = [] for r in related_repos: # Don't include the repo being updated in this list if r['id'] == repo_id: continue all_configs = [d['config'] for d in r['distributors']] trr = common_utils.to_related_repo(r, all_configs) transfer_related_repos.append(trr) try: result = distributor_instance.validate_config( transfer_repo, call_config, transfer_related_repos) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: _LOG.exception( 'Exception raised from distributor [%s] while validating config for repo [%s]' % (distributor_type_id, repo_id)) raise PulpDataException(e.args), None, sys.exc_info()[2]
def test_valid_str(self): config = PluginCallConfiguration({}, {'a': 'true'}) importer_config._run_validate_is_non_required_bool(config, 'a')
def test_optional(self): config = PluginCallConfiguration({}, {}) importer_config.validate_retain_old_count(config)
def add_distributor(self, repo_group_id, distributor_type_id, group_plugin_config, distributor_id=None): """ Adds an association from the given repository group to a distributor. The assocation will be tracked through the distributor_id; each distributor on a given group must have a unique ID. If this is not specified, one will be generated. If a distributor already exists on the group with a given ID, the existing one will be removed and replaced with the newly configured one. @param repo_group_id: identifies the repo group @type repo_group_id: str @param distributor_type_id: type of distributor being added; must reference one of the installed group distributors @type distributor_type_id: str @param group_plugin_config: config to use for the distributor for this group alone @type group_plugin_config: dict @param distributor_id: if specified, the newly added distributor will be referenced by this value and the group id; if omitted one will be generated @type distributor_id: str @return: database representation of the added distributor @rtype: dict @raise MissingResource: if the group doesn't exist @raise InvalidValue: if a distributor ID is provided and is not valid @raise PulpDataException: if the plugin indicates the config is invalid @raise PulpExecutionException: if the plugin raises an exception while initializing the newly added distributor """ distributor_coll = RepoGroupDistributor.get_collection() query_manager = manager_factory.repo_group_query_manager() # Validation group = query_manager.get_group( repo_group_id) # will raise MissingResource if not plugin_api.is_valid_group_distributor(distributor_type_id): raise InvalidValue(['distributor_type_id']) # Determine the ID for the distributor on this repo if distributor_id is None: distributor_id = str(uuid.uuid4()) else: # Validate if one was passed in if not is_distributor_id_valid(distributor_id): raise InvalidValue(['distributor_id']) distributor_instance, plugin_config = plugin_api.get_group_distributor_by_id( distributor_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. clean_config = None if group_plugin_config is not None: clean_config = dict([(k, v) for k, v in group_plugin_config.items() if v is not None]) # Let the plugin validate the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_group = common_utils.to_transfer_repo_group(group) transfer_group.working_dir = common_utils.distributor_working_dir( distributor_type_id, repo_group_id) # Load the related groups which is needed for the validation transfer_related_groups = related_groups(distributor_type_id) # Request the plugin validate the configuration try: is_valid, message = distributor_instance.validate_config( transfer_group, call_config, transfer_related_groups) if not is_valid: raise PulpDataException(message) except Exception, e: _LOG.exception( 'Exception received from distributor [%s] while validating config' % distributor_type_id) raise PulpDataException(e.args), None, sys.exc_info()[2]